/*
 * Copyright (c) 2016. For Intelligent Group.
 */

package com.intelligent.ispc.core.service.impl;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.intelligent.ispc.cache.GrrDataCacheFactory;
import com.intelligent.ispc.cache.UserDataCacheFactory;
import com.intelligent.ispc.chart.service.impl.ChartExporterImpl;
import com.intelligent.ispc.common.mapper.JsonMapper;
import com.intelligent.ispc.common.utils.*;
import com.intelligent.ispc.constant.AppConstant;
import com.intelligent.ispc.constant.ProcessConstant;
import com.intelligent.ispc.core.dto.*;
import com.intelligent.ispc.core.service.GrrService;
import com.intelligent.ispc.core.service.SysConfigService;
import com.intelligent.ispc.core.service.SystemService;
import com.intelligent.ispc.export.grr.GrrExportWorker;
import com.intelligent.ispc.r.service.AnalysisService;
import com.intelligent.ispc.utils.ApplicationPathUtil;
import com.intelligent.ispc.utils.FileUtil;
import com.intelligent.ispc.utils.ProcessHelper;
import com.intelligent.ispc.utils.PythonUtil;
import com.intelligent.ispc.utils.executor.ExportThreadManager;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.File;

import com.intelligent.ispc.export.grr.GrrExcelBuilder;

import java.io.FileWriter;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;

/**
 * Created by Julia.Zhou on 2016/08/16.
 */
@Service
public class GrrServiceImpl implements GrrService {

    private final Logger logger = LoggerFactory.getLogger(GrrServiceImpl.class);

    @Autowired
    private SystemService systemService;

    @Autowired
    private AnalysisService analysisService;

    @Autowired
    private SysConfigService sysConfigService;

    @Autowired
    private GrrDataCacheFactory grrDataCacheFactory;

    @Autowired
    private UserDataCacheFactory userDataCacheFactory;

    private JsonMapper mapper = JsonMapper.nonDefaultMapper();

    @Override
    public GrrGroupDto searchGroupSearchKeys(String filePath) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.GRR_DECODE_FILE);
        if (StringUtils.isBlank(filePath)) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PARAMETER_INVALID));
        }
        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING));
        //clear cache
        grrDataCacheFactory.getCacheGrrViewData().invalidateAll();

        //get required configuration
        GrrConfigDto grrConfigDto = systemService.findGrrConfig();
        String fileName = filePath.substring(filePath.lastIndexOf(File.separator) + 1, filePath.length());
        GrrItemDto grrItemDto = systemService.findGrrItemByGrrName(fileName);
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();

        //validate and set params
        validateParams(grrConfigDto, grrItemDto, null, systemConfigDto);

        GrrGroupDto grrGroupDto = new GrrGroupDto();
        grrGroupDto.setGroupkey1(grrConfigDto.getGroup1Key());
        grrGroupDto.setGroupkey2(grrConfigDto.getGroup2Key());

        Set<String> groupValues = new LinkedHashSet<>();
        Map<String, Boolean> resultItemNames = new LinkedHashMap<>();
        List<String> filePaths = Lists.newArrayList();
        filePaths.add(filePath);
        String scriptPath = this.getScriptPath();

        //get source data
        Map<String, List<String>> testDatas = this.getDataByName(filePaths, scriptPath, null);

        processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_DONE));
        processHelper.postProcess(21, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_CONDITION));
        //get group values
        if (testDatas != null && !testDatas.isEmpty()) {
            for (String mapKey : testDatas.keySet()) {
                if (mapKey.contains(AppConstant.DATA_INFO)) {
                    List<Map> dataList = mapper.fromJson(testDatas.get(mapKey).toString(), List.class);
                    for (int i = 0; i < dataList.size(); i++) {
                        Map<String, String> row = dataList.get(i);
                        if (StringUtils.isBlank(row.get(grrConfigDto.getPartKey()))) {
                            continue;
                        }
                        String groupValue = null;
                        if (StringUtils.isNotBlank(grrConfigDto.getGroup1Key()) && StringUtils.isBlank(grrConfigDto.getGroup2Key())) {
                            groupValue = row.get(grrConfigDto.getGroup1Key());
                        } else if (StringUtils.isBlank(grrConfigDto.getGroup1Key()) && StringUtils.isNotBlank(grrConfigDto.getGroup2Key())) {
                            groupValue = row.get(grrConfigDto.getGroup2Key());
                        } else if (StringUtils.isNotBlank(grrConfigDto.getGroup1Key()) && StringUtils.isNotBlank(grrConfigDto.getGroup2Key())) {
                            groupValue = row.get(grrConfigDto.getGroup1Key()) + AppConstant.GLOBAL_SEPARATOR + row.get(grrConfigDto.getGroup2Key());
                        }
                        groupValues.add(groupValue);
                    }
                    dataList = null;
                }
            }
        }
        //get search and attribute keys
        Set<String> searchAndAttributeKeys = this.getSearchAndAttributeKeys(filePaths, scriptPath, systemConfigDto);
        processHelper.postProcess(30, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_CONDITION_DONE));
        processHelper.postProcess(31, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM));
        processHelper.postProcess(40, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_VALIDATE));

        //get source itemNames
        List<String> itemNames = this.getSourceTestItemNames(filePaths, scriptPath);
        itemNames = this.getTestItemKeys(itemNames);
        PythonUtil pythonUtil = new PythonUtil();

        //get source all column names
        List<String> allColumnNames = getAllColNames(mapper.fromJson(pythonUtil.getColumnKeys(filePaths, false).toString(), List.class));

        String[] testStrategy = null;
        List<String> groupResultValues = Lists.newArrayList();
        //Continue processing group values
        if (AppConstant.GRR_MODEL_SLOT.equals(grrConfigDto.getScheme())) {
            if (StringUtils.isNotBlank(grrItemDto.getDimension())) {
                testStrategy = grrItemDto.getDimension().split(";");
                if (testStrategy != null && groupValues != null) {
                    for (String groupValue : groupValues) {
                        for (int z = 0; z < testStrategy.length; z++) {
                            groupResultValues.add(groupValue + AppConstant.GLOBAL_SEPARATOR + testStrategy[z]);
                        }
                    }
                    grrGroupDto.setValue(groupResultValues);
                }
                groupResultValues = null;
                testStrategy = null;
            }
        } else {
            List<String> groups = Lists.newLinkedList();
            groups.addAll(groupValues);
            grrGroupDto.setValue(groups);
            groups = null;
        }

        String errorKeys = "";
        //get last testItem and erify the keys of the configuration exists in the file
        if (searchAndAttributeKeys != null && !searchAndAttributeKeys.isEmpty()) {
            for (String key : searchAndAttributeKeys) {
                itemNames.remove(key);
                if (!allColumnNames.contains(key)) {
                    errorKeys += key + System.getProperty("line.separator");
                }
            }
        }

        if (!allColumnNames.contains(grrConfigDto.getPartKey()) && !errorKeys.contains(grrConfigDto.getPartKey())) {
            errorKeys += StringUtils.isBlank(grrConfigDto.getPartKey()) ? "" : (grrConfigDto.getPartKey() + System.getProperty("line.separator"));
        }

        if (!allColumnNames.contains(grrConfigDto.getGroup1Key()) && !errorKeys.contains(grrConfigDto.getGroup1Key())) {
            errorKeys += StringUtils.isBlank(grrConfigDto.getGroup1Key()) ? "" : (grrConfigDto.getGroup1Key() + System.getProperty("line.separator"));
        }

        if (!allColumnNames.contains(grrConfigDto.getGroup2Key()) && !errorKeys.contains(grrConfigDto.getGroup2Key())) {
            errorKeys += StringUtils.isBlank(grrConfigDto.getGroup2Key()) ? "" : (grrConfigDto.getGroup2Key() + System.getProperty("line.separator"));
        }

        if (!allColumnNames.contains(grrConfigDto.getOperatorKey()) && !errorKeys.contains(grrConfigDto.getOperatorKey())) {
            errorKeys += StringUtils.isBlank(grrConfigDto.getOperatorKey()) ? "" : (grrConfigDto.getOperatorKey() + System.getProperty("line.separator"));
        }

        if (StringUtils.isNotBlank(errorKeys)) {
            String[] args = new String[]{errorKeys};
            throw new ApplicationException(ExceptionMessages.ERR_12002, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_COLUMN_INVALID, args));
        }

        this.validateTestDatas(filePath, testDatas, grrConfigDto, grrItemDto, groupResultValues);
        if (AppConstant.GRR_MODEL_SLOT.equals(grrConfigDto.getScheme()) && (grrGroupDto.getValue() == null || grrGroupDto.getValue().isEmpty())) {
            grrGroupDto.setValue(groupResultValues);
        }

        //The first ten items for the selected default setting
        if (allColumnNames != null && !allColumnNames.isEmpty()) {
            final int defaultNumber = 9;
            int defaultCheckedTestItemName = 0;
            for (String key : allColumnNames) {
                if (itemNames.contains(key)) {
                    if (defaultCheckedTestItemName <= defaultNumber) {
                        resultItemNames.put(key, true);
                    } else {
                        resultItemNames.put(key, false);
                    }
                    defaultCheckedTestItemName++;
                }
            }
            grrItemDto.setTestItemNames(mapper.toJson(resultItemNames));
        }
        processHelper.postProcess(45, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_PARSING_ITEM_DONE));
        systemService.updateGrrItem(grrItemDto);
        allColumnNames = null;
        searchAndAttributeKeys = null;
        itemNames = null;
        grrItemDto = null;
        groupValues = null;
        resultItemNames = null;
        grrConfigDto = null;
        testDatas = null;
        grrItemDto = null;
        processHelper.postProcess(50, ResourceBundleUtils.getString(AppConstant.LOG_PARSING_FILE_VALIDATE_DONE));
        return grrGroupDto;
    }

    @Override
    @Deprecated
    public GrrMainResultDto searchGrrMainResult(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey) {
        GrrMainResultDto grrMainResultDto = new GrrMainResultDto();
        if (grrParamDto == null || StringUtils.isBlank(grrParamDto.getFilePath()) || groupSearchKey == null || StringUtils.isBlank(groupSearchKey.getGroupkey1())) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PARAMETER_INVALID));
        }
        String filePath = grrParamDto.getFilePath();
        //get required configuration
        GrrConfigDto grrConfigDto = systemService.findGrrConfig();
        String fileName = filePath.substring(filePath.lastIndexOf(File.separator) + 1, filePath.length());
        GrrItemDto grrItemDto = systemService.findGrrItemByGrrName(fileName);
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();

        //validate and set params
        this.validateParams(grrConfigDto, grrItemDto, grrParamDto, systemConfigDto);

        //parse group
        groupSearchKey = parseGrrGroup(groupSearchKey);

        List<String> filePaths = Lists.newArrayList();
        filePaths.add(filePath);
        String scriptPath = this.getScriptPath();
        Map<String, List<String>> testDatas = null;
        List<String> itemNameCheckedLists = null;
        //get need testItems
        if (StringUtils.isBlank(grrParamDto.getItemName())) {
            String itemNameFilePath = ApplicationPathUtil.getCanonicalPath() + "testItemNames.txt";
            itemNameCheckedLists = this.getTestItemNames(grrItemDto, itemNameFilePath, -1, -1);
            testDatas = this.getDataByFile(filePaths, scriptPath, itemNameFilePath);
            FileUtils.deleteFile(itemNameFilePath);
        } else {
            itemNameCheckedLists = Lists.newLinkedList();
            itemNameCheckedLists.add(grrParamDto.getItemName());
            testDatas = this.getDataByName(filePaths, scriptPath, itemNameCheckedLists);
        }

        //get grr view data
        List<GrrViewDataDto> grrViewDataDtos = this.getGrrViewDatas(grrParamDto, groupSearchKey, itemNameCheckedLists, grrConfigDto, testDatas);
        grrMainResultDto.setGrrViewDataDtos(grrViewDataDtos);

        //get grr summary
        List<GrrSummaryDto> grrSummaryDtos = this.getGrrSummary(grrParamDto, grrViewDataDtos, this.getUslAndLslAndUnits(filePaths, scriptPath));
        grrMainResultDto.setGrrSummaryDtos(grrSummaryDtos);

        grrSummaryDtos = null;
        grrViewDataDtos = null;
        filePaths = null;
        testDatas = null;
        return grrMainResultDto;
    }

    @Override
    public List<GrrSummaryDto> searchGrrSummary(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey, int startIndex, int endIndex) {
        ProcessHelper decodeFileHelper = new ProcessHelper(ProcessConstant.GRR_DECODE_FILE);
        List<String> filePaths = Lists.newArrayList();
        filePaths.add(grrParamDto.getFilePath());
        String scriptPath = this.getScriptPath();
        decodeFileHelper.postProcess(65, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_SUMMARY));

        //get usl, lsl, unit all testItem
        Map<String, Object> uslAndLslAndUnits = this.getUslAndLslAndUnits(filePaths, scriptPath);

        //get grr view data
        List<GrrViewDataDto> grrViewDataDtos = this.searchGrrViewData(grrParamDto, groupSearchKey, startIndex, endIndex);

        //get grr summary
        List<GrrSummaryDto> grrSummaryDtos = this.getGrrSummary(grrParamDto, grrViewDataDtos, uslAndLslAndUnits);

        grrViewDataDtos = null;
        uslAndLslAndUnits = null;
        filePaths = null;
        decodeFileHelper.postProcess(90, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_SUMMARY_DONE));
        decodeFileHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_FINISH));
        return grrSummaryDtos;
    }

    @Override
    public List<GrrViewDataDto> searchGrrViewData(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey, int startIndex, int endIndex) {
        if (grrParamDto == null || StringUtils.isBlank(grrParamDto.getFilePath()) || groupSearchKey == null || StringUtils.isBlank(groupSearchKey.getGroupkey1())) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PARAMETER_INVALID));
        }
        String filePath = grrParamDto.getFilePath();
        //get required configuration
        GrrConfigDto grrConfigDto = systemService.findGrrConfig();
        String fileName = filePath.substring(filePath.lastIndexOf(File.separator) + 1, filePath.length());
        GrrItemDto grrItemDto = systemService.findGrrItemByGrrName(fileName);
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();

        //validate and set params
        this.validateParams(grrConfigDto, grrItemDto, grrParamDto, systemConfigDto);

        groupSearchKey = this.parseGrrGroup(groupSearchKey);

        String itemNameFilePath = ApplicationPathUtil.getCanonicalPath() + "testItemNames.txt";
        //write need testItems to file
        List<String> itemNameCheckedLists = this.getTestItemNames(grrItemDto, itemNameFilePath, startIndex, endIndex);

        List<String> filePaths = Lists.newArrayList();
        filePaths.add(filePath);
        String scriptPath = this.getScriptPath();

        //get need testItems
        Map<String, List<String>> testDatas = this.getDataByFile(filePaths, scriptPath, itemNameFilePath);

        //get grr view data
        List<GrrViewDataDto> grrViewDataDtos = this.getGrrViewDatas(grrParamDto, groupSearchKey, itemNameCheckedLists, grrConfigDto, testDatas);
        FileUtils.deleteFile(itemNameFilePath);
        testDatas = null;
        filePaths = null;
        itemNameCheckedLists = null;
        return grrViewDataDtos;
    }

    @Override
    public GrrResultDto searchGrrResult(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey, List<GrrViewDataDto> grrViewDataDtos) {
        if (grrParamDto == null || StringUtils.isBlank(grrParamDto.getFilePath())) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_PARAMETER_INPUT_INVALID));
        }
        String filePath = grrParamDto.getFilePath();
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.GRR_SEARCH_SUB_TASK);
        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_VALIDATE));

        //get grrConfig
        GrrConfigDto grrConfigDto = systemService.findGrrConfig();
        String fileName = filePath.substring(filePath.lastIndexOf(File.separator) + 1, filePath.length());

        //get grrItem
        GrrItemDto grrItemDto = systemService.findGrrItemByGrrName(fileName);
        SystemConfigDto systemConfigDto = systemService.findSystemConfig();

        //validate and set params
        this.validateParams(grrConfigDto, grrItemDto, grrParamDto, systemConfigDto);

        processHelper.postProcess(15, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_VALIDATE_DONE));
        processHelper.postProcess(17, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_ITEM_DETAIL));

        //filter itemName
        List<String> filePaths = Lists.newArrayList();
        filePaths.add(grrParamDto.getFilePath());
        String scriptPath = this.getScriptPath();

        List<String> itemNameCheckedLists = Lists.newLinkedList();
        itemNameCheckedLists.add(grrParamDto.getItemName());

        //get source data of filter
        Map<String, List<String>> testDatas = this.getDataByName(filePaths, scriptPath, itemNameCheckedLists);

        //get processed data
        List<GrrViewDataDto> grrViewDataDtos1 = this.getGrrViewDatas(grrParamDto, groupSearchKey, itemNameCheckedLists, grrConfigDto, testDatas);

        //get a testItem data
        GrrResultDto resultDto = this.getSubGRRResult(grrParamDto, grrViewDataDtos1);

        processHelper.postProcess(35, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_ITEM_DETAIL_DONE));
        processHelper.postProcess(40, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_ANALYSIS));

        //get usl, lsl, unit the testItem
        Map<String, Object> uslAndLslAndUnits = this.getUslAndLslAndUnits(filePaths, scriptPath);

        //set data of R
        GrrAnalysisDataDto grrAnalysisDataDto = this.getGrrAnalysisData(grrParamDto, grrViewDataDtos1, uslAndLslAndUnits);
        processHelper.postProcess(50, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_ANALYSIS_DONE));
        processHelper.postProcess(55, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_ANOVA_SOURCE));

        //get anova and source
        Map<String, Object> grrAnovaDtoAndSourceDto = analysisService.analyseGrrAnovaAndSource(grrParamDto, grrAnalysisDataDto);
        List<GrrAnovaDto> grrAnovaDtoList = (List<GrrAnovaDto>) grrAnovaDtoAndSourceDto.get(AppConstant.GRR_ANOVA_VALUE);
        List<GrrSourceDto> grrSourceDtoList = (List<GrrSourceDto>) grrAnovaDtoAndSourceDto.get(AppConstant.GRR_SOURCE_VALUE);

        processHelper.postProcess(70, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_ANOVA_SOURCE_DONE));
        processHelper.postProcess(75, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_CHARTS));
        //get chart data
        GrrChartDto grrChartDto = analysisService.analyseGrrChartData(grrParamDto, grrAnalysisDataDto);
        processHelper.postProcess(90, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_CHARTS_DONE));

        resultDto.setGrrAnovaDtoList(grrAnovaDtoList);
        resultDto.setGrrSourceDtoList(grrSourceDtoList);
        resultDto.setGrrChartDto(grrChartDto);
        grrChartDto = null;
        grrSourceDtoList = null;
        grrAnovaDtoList = null;
        grrParamDto = null;
        grrViewDataDtos = null;
        grrViewDataDtos1 = null;
        grrAnovaDtoAndSourceDto = null;
        filePaths = null;
        uslAndLslAndUnits = null;
        processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_GRR_EXECUTE_FINISH));
        return resultDto;
    }

    @Override
    public String exportSummary(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey, List<String> itemNames) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.GRR_EXPORT_FILE);
        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE));
        if (grrParamDto == null || groupSearchKey == null) {
            throw new ApplicationException(ExceptionMessages.ERR_12010, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_EXPORT_PARAMETER_INVALID));
        }
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE_DONE));
        processHelper.postProcess(15, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_DISENV));
        String[] exportFilePath = new String[1];
        try {
            processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_START_EXPORT));
            boolean ifSucceed = false;
            ExportThreadManager exportThreadManager = ExportThreadManager.getExportThreadManager();

            Callable<Boolean> callableTaskOne = () -> exportSummary(grrParamDto, groupSearchKey, itemNames, exportFilePath, processHelper, exportThreadManager);
            FutureTask<Boolean> futureTaskOne = exportThreadManager.initGrrExportThread("TaskOne", callableTaskOne);
            exportThreadManager.startGrrExportThread();
            try {
                ifSucceed = futureTaskOne.get();
            } catch (Exception e) {
                futureTaskOne.cancel(true);
                logger.error("Export grr only result failed!", e.getMessage());
                throw new ApplicationException(ExceptionMessages.ERR_20001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GLOBAL_SYSTEM));
            }
            if (ifSucceed) {
                processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FINISH) + exportFilePath[0]);
            } else {
                processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FAIL));
            }
        } catch (ApplicationException e) {
            logger.error("Export grr only result failed!", e);
            throw new ApplicationException(e.getCode(), e.getMessage());
        }
        return exportFilePath[0];
    }

    @Override
    public String exportSummaryAndDetail(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey, List<String> itemNames) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.GRR_EXPORT_FILE);
        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE));
        if (grrParamDto == null || groupSearchKey == null) {
            throw new ApplicationException(ExceptionMessages.ERR_12010, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_EXPORT_PARAMETER_INVALID));
        }
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_VALIDATE_DONE));
        String[] exportFilePath = new String[1];
        try {
            boolean ifSucceed = false;
            ExportThreadManager exportThreadManager = ExportThreadManager.getExportThreadManager();
            Callable<Boolean> callableTaskOne = () -> exportSummaryAndDetail(grrParamDto, groupSearchKey, itemNames, exportFilePath, processHelper, exportThreadManager);
            FutureTask<Boolean> futureTaskOne = exportThreadManager.initGrrExportThread("TaskOne", callableTaskOne);
            exportThreadManager.startGrrExportThread();
            try {
                ifSucceed = futureTaskOne.get();
            } catch (Exception e) {
                futureTaskOne.cancel(true);
                logger.error("Export grr result and data failed!", e.getMessage());
                throw new ApplicationException(ExceptionMessages.ERR_20001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GLOBAL_SYSTEM));
            }
            if (ifSucceed) {
                processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FINISH) + exportFilePath[0]);
            } else {
                processHelper.postProcess(100, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_FAIL));
            }
        } catch (ApplicationException e) {
            logger.error("Export grr result and data failed!", e);
            throw new ApplicationException(e.getCode(), e.getMessage());
        }
        return exportFilePath[0];
    }

    @Override
    public void updateGrrViewDatas(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey) {
        ProcessHelper processHelper = new ProcessHelper(ProcessConstant.GRR_UPDATE_TASK);
        processHelper.postProcess(5, ResourceBundleUtils.getString(AppConstant.LOG_GRR_UPDATE_PARAMETER_VALIDATE));
        if (grrParamDto == null || StringUtils.isBlank(grrParamDto.getFilePath()) || groupSearchKey == null || StringUtils.isBlank(groupSearchKey.getGroupkey1())) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PARAMETER_INVALID));
        }
        processHelper.postProcess(10, ResourceBundleUtils.getString(AppConstant.LOG_GRR_UPDATE_PARAMETER_VALIDATE_DONE));
        String key = grrParamDto.getFilePath() + groupSearchKey;
        Map<String, Set<String>> result = new LinkedHashMap<>();
        Map<String, Set<String>> changedGrrViewData = null;

        processHelper.postProcess(15, ResourceBundleUtils.getString(AppConstant.LOG_GRR_UPDATE_DATA));
        try {
            //get cache grr view data
            String grrViewData = grrDataCacheFactory.getCacheGrrViewData().get(grrParamDto.getFilePath() + groupSearchKey);
            if (StringUtils.isNotBlank(grrViewData) && !grrViewData.equals("[]")) {
                changedGrrViewData = mapper.fromJson(grrViewData, mapper.contructMapType(Map.class, String.class, Set.class));
            }
            Set<String> oldIncludeIds = new HashSet<>();
            Set<String> oldExcludeIds = new HashSet<>();
            if (changedGrrViewData != null && !changedGrrViewData.isEmpty()) {
                oldIncludeIds = changedGrrViewData.get("includeIds");
                oldExcludeIds = changedGrrViewData.get("excludeIds");
            }
            List<String> newIncludeIds = grrParamDto.getNewIncludeIds();
            List<String> newExcludeIds = grrParamDto.getNewExcludeIds();

            Set<String> oldItemValueIncludeIdList = new HashSet<>();
            Set<String> oldItemValueExcludeIdList = new HashSet<>();
            if (newIncludeIds != null && newExcludeIds != null) {
                if (oldExcludeIds != null && !oldExcludeIds.isEmpty()) {
                    oldIncludeIds.addAll(newIncludeIds);
                    oldExcludeIds.addAll(newExcludeIds);
                    Set<String> ignore = new HashSet<>();
                    for (String in : oldIncludeIds) {
                        if (oldExcludeIds.contains(in)) {
                            ignore.add(in);
                        }
                    }
                    oldExcludeIds.removeAll(ignore);
                    oldIncludeIds.removeAll(ignore);

                    oldItemValueIncludeIdList.addAll(oldIncludeIds);
                    oldItemValueExcludeIdList.addAll(oldExcludeIds);
                } else {
                    oldItemValueIncludeIdList.addAll(newIncludeIds);
                    oldItemValueExcludeIdList.addAll(newExcludeIds);
                }
                if (!oldItemValueIncludeIdList.isEmpty()) {
                    result.put("includeIds", oldItemValueIncludeIdList);
                    result.put("excludeIds", oldItemValueExcludeIdList);
                }
            }
            oldItemValueIncludeIdList = null;
            oldItemValueExcludeIdList = null;
            newIncludeIds = null;
            newExcludeIds = null;
            oldIncludeIds = null;
            oldExcludeIds = null;
            changedGrrViewData = null;
        } catch (ExecutionException e) {
            throw new ApplicationException(ExceptionMessages.ERR_20001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GLOBAL_SYSTEM));
        }
        //set latest result to cache
        grrDataCacheFactory.getCacheGrrViewData().put(key, mapper.toJson(result));
        processHelper.postProcess(50, ResourceBundleUtils.getString(AppConstant.LOG_GRR_UPDATE_DATA_DONE));
        result = null;
    }

    /*
     * parse grr group
     */
    private GrrGroupDto parseGrrGroup(GrrGroupDto groupSearchKey) {
        GrrGroupDto grrGroupDto = groupSearchKey;
        List<String> value = groupSearchKey.getValue();
        if (value != null && !value.isEmpty()) {
            if (StringUtils.isNotBlank(grrGroupDto.getGroupkey1()) && StringUtils.isNotBlank(grrGroupDto.getGroupkey2())) {
                if (!value.get(0).contains(AppConstant.GLOBAL_SEPARATOR)) {
                    grrGroupDto.setGroupValue1(value.get(0));
                } else {
                    String[] values = value.get(0).split(AppConstant.GLOBAL_SEPARATOR);
                    if (values.length == 2) {
                        grrGroupDto.setGroupValue1(values[0]);
                        grrGroupDto.setGroupValue2(values[1]);
                    }
                    if (values.length > 2) {
                        grrGroupDto.setGroupValue1(values[0]);
                        grrGroupDto.setGroupValue2(values[1]);
                        grrGroupDto.setDimensionValue(values[2]);
                    }
                    values = null;
                }
            } else if (StringUtils.isNotBlank(grrGroupDto.getGroupkey1()) && StringUtils.isBlank(grrGroupDto.getGroupkey2())) {
                if (!value.get(0).contains(AppConstant.GLOBAL_SEPARATOR)) {
                    grrGroupDto.setGroupValue1(value.get(0));
                } else {
                    String[] values = value.get(0).split(AppConstant.GLOBAL_SEPARATOR);
                    if (values != null && values.length > 1) {
                        grrGroupDto.setGroupValue1(values[0]);
                        grrGroupDto.setDimensionValue(values[1]);
                    }
                    values = null;
                }
            } else if (StringUtils.isBlank(grrGroupDto.getGroupkey1()) && StringUtils.isBlank(grrGroupDto.getGroupkey2())) {
                if (!value.get(0).contains(AppConstant.GLOBAL_SEPARATOR)) {
                    grrGroupDto.setDimensionValue(value.get(0));
                }
            }
        }
        value = null;
        return grrGroupDto;
    }

    private List<GrrViewDataDto> parseGrrNormalForTrial(Map<String, List<Map<String, String>>> primaryMap, String itemNameTemp,
                                                        Integer partValue, Integer operatorValue, Integer trialValue, Map<String, Set<String>> changedGrrViewData) {
        List<GrrViewDataDto> result = Lists.newLinkedList();
        List<GrrViewDataDto> grrNormalList = Lists.newLinkedList();
        List<GrrViewDataDto> grrMoreList = Lists.newLinkedList();

        Set<String> oldIncludeIds = new HashSet<>();
        Set<String> oldExcludeIds = new HashSet<>();
        //get cache ids
        if (changedGrrViewData != null && !changedGrrViewData.isEmpty()) {
            oldIncludeIds = changedGrrViewData.get("includeIds");
            oldExcludeIds = changedGrrViewData.get("excludeIds");
        }
        boolean isMore = false;
        Integer total = operatorValue * trialValue * partValue;
        //process data
        for (String key : primaryMap.keySet()) {
            List<Map<String, String>> itemOfParts = primaryMap.get(key);
            Integer trial = 1, operator = 1, k = 1;
            for (Map<String, String> itemOfPart : itemOfParts) {
                GrrViewDataDto grrViewDataDto = new GrrViewDataDto();
                grrViewDataDto.setPart(key);

                List<KeyValueDto> testItemNameList = Lists.newLinkedList();
                List<String> list1 = new LinkedList<>(itemOfPart.keySet());
                grrViewDataDto.setId(list1.get(itemOfPart.keySet().size() - 1));
                list1 = null;
                //get testItem data
                for (String itemName : itemOfPart.keySet()) {
                    if (grrViewDataDto.getId().equals(itemName)) {
                        continue;
                    }
                    if (StringUtils.isNotBlank(itemNameTemp)) {
                        if (itemNameTemp.equals(itemName)) {
                            KeyValueDto baseDtoResult = new KeyValueDto();
                            baseDtoResult.setKey(itemName);
                            baseDtoResult.setValue(StringUtils.formatBigDecimal(itemOfPart.get(itemName)));
                            testItemNameList.add(baseDtoResult);
                            baseDtoResult = null;
                            break;
                        }
                    } else {
                        KeyValueDto keyValueDto = new KeyValueDto();
                        keyValueDto.setKey(itemName);
                        keyValueDto.setValue(StringUtils.formatBigDecimal(itemOfPart.get(itemName)));
                        testItemNameList.add(keyValueDto);
                        keyValueDto = null;
                    }
                }
                //replace related data
                if (oldExcludeIds.contains(grrViewDataDto.getId())) {
                    grrViewDataDto.setOperator("");
                    grrViewDataDto.setTrial("");
                    isMore = true;
                } else {
                    if (trial != null && trial == trialValue + 1) {
                        operator++;
                        trial = 1;
                    }
                    grrViewDataDto.setOperator(operator.toString());
                    grrViewDataDto.setTrial(trial.toString());
                    if (k - (operatorValue * trialValue) > 0) {
                        if (!oldIncludeIds.contains(grrViewDataDto.getId())) {
                            grrViewDataDto.setOperator("");
                            grrViewDataDto.setTrial("");
                            isMore = true;
                        }
                    }
                }
                grrViewDataDto.setColValueDtoList(testItemNameList);
                k++;
                //check need data
                if (grrNormalList.size() >= total) {
                    grrViewDataDto.setOperator("");
                    grrViewDataDto.setTrial("");
                    grrViewDataDto.setCheck(false);
                    grrMoreList.add(grrViewDataDto);
                    isMore = false;
                } else {
                    if (isMore) {
                        grrViewDataDto.setCheck(false);
                        grrMoreList.add(grrViewDataDto);
                        isMore = false;
                    } else {
                        trial++;
                        grrViewDataDto.setCheck(true);
                        grrNormalList.add(grrViewDataDto);
                    }
                }
                grrViewDataDto = null;
                testItemNameList = null;
            }
            itemOfParts = null;
        }
        result.addAll(grrNormalList);
        result.addAll(grrMoreList);
        grrNormalList = null;
        grrMoreList = null;
        oldIncludeIds = null;
        oldExcludeIds = null;
        return result;
    }

    private List<GrrViewDataDto> parseGrrNormalForOperator(Map<String, List<Map<String, String>>> primaryMap, String itemNameTemp, Integer partValue,
                                                           Integer operatorValue, Integer trialValue, Map<String, Set<String>> changedGrrViewData) {
        List<GrrViewDataDto> result = Lists.newLinkedList();
        List<GrrViewDataDto> grrNormalList = Lists.newLinkedList();
        List<GrrViewDataDto> grrMoreList = Lists.newLinkedList();

        Set<String> oldIncludeIds = new HashSet<>();
        Set<String> oldExcludeIds = new HashSet<>();
        //get cache ids
        if (changedGrrViewData != null && !changedGrrViewData.isEmpty()) {
            oldIncludeIds = changedGrrViewData.get("includeIds");
            oldExcludeIds = changedGrrViewData.get("excludeIds");
        }
        boolean isMore = false;
        Integer total = operatorValue * trialValue * partValue;
        //process data
        for (String key : primaryMap.keySet()) {
            List<Map<String, String>> itemOfParts = primaryMap.get(key);
            Integer operator = 1, trial = 1, k = 1;
            for (Map<String, String> itemOfPart : itemOfParts) {
                GrrViewDataDto grrViewDataDto = new GrrViewDataDto();
                grrViewDataDto.setPart(key);

                List<KeyValueDto> testItemNameList = Lists.newLinkedList();
                List<String> list1 = new LinkedList<>(itemOfPart.keySet());
                grrViewDataDto.setId(list1.get(itemOfPart.keySet().size() - 1));
                list1 = null;

                //get testItem data
                for (String itemName : itemOfPart.keySet()) {
                    if (grrViewDataDto.getId().equals(itemName)) {
                        continue;
                    }
                    if (StringUtils.isNotBlank(itemNameTemp)) {
                        if (itemNameTemp.equals(itemName)) {
                            KeyValueDto baseDtoResult = new KeyValueDto();
                            baseDtoResult.setKey(itemName);
                            baseDtoResult.setValue(StringUtils.formatBigDecimal(itemOfPart.get(itemName)));
                            testItemNameList.add(baseDtoResult);
                            baseDtoResult = null;
                            break;
                        }
                    } else {
                        KeyValueDto keyValueDto = new KeyValueDto();
                        keyValueDto.setKey(itemName);
                        keyValueDto.setValue(StringUtils.formatBigDecimal(itemOfPart.get(itemName)));
                        testItemNameList.add(keyValueDto);
                        keyValueDto = null;
                    }
                }
                //replace related data
                if (oldExcludeIds.contains(grrViewDataDto.getId())) {
                    grrViewDataDto.setOperator("");
                    grrViewDataDto.setTrial("");
                    isMore = true;
                } else {
                    if (operator != null && operator == operatorValue + 1) {
                        trial++;
                        operator = 1;
                    }
                    grrViewDataDto.setOperator(operator.toString());
                    grrViewDataDto.setTrial(trial.toString());
                    if (k - (operatorValue * trialValue) > 0) {
                        if (!oldIncludeIds.contains(grrViewDataDto.getId())) {
                            grrViewDataDto.setOperator("");
                            grrViewDataDto.setTrial("");
                            isMore = true;
                        }
                    }
                }
                grrViewDataDto.setColValueDtoList(testItemNameList);
                k++;
                //check need data
                if (grrNormalList.size() >= total) {
                    grrViewDataDto.setOperator("");
                    grrViewDataDto.setTrial("");
                    grrViewDataDto.setCheck(false);
                    grrMoreList.add(grrViewDataDto);
                    isMore = false;
                } else {
                    if (isMore) {
                        grrViewDataDto.setCheck(false);
                        grrMoreList.add(grrViewDataDto);
                        isMore = false;
                    } else {
                        operator++;
                        grrViewDataDto.setCheck(true);
                        grrNormalList.add(grrViewDataDto);
                    }
                }
                grrViewDataDto = null;
                testItemNameList = null;
            }
            itemOfParts = null;
        }
        //change first operator data structure to first trial data structure for normal
        Map<String, List<GrrViewDataDto>> snAndOperatorMap = new LinkedHashMap<>();
        for (GrrViewDataDto grrDataDto : grrNormalList) {
            String snAndOp = grrDataDto.getPart() + grrDataDto.getOperator();
            if (!snAndOperatorMap.containsKey(snAndOp)) {
                List<GrrViewDataDto> grrDataDtoList1 = Lists.newLinkedList();
                grrDataDtoList1.add(grrDataDto);
                snAndOperatorMap.put(snAndOp, grrDataDtoList1);
                grrDataDtoList1 = null;
            } else {
                snAndOperatorMap.get(snAndOp).add(grrDataDto);
            }
        }
        for (String key : snAndOperatorMap.keySet()) {
            result.addAll(snAndOperatorMap.get(key));
        }
        //change first operator data structure to first trial data structure for more
        if (grrMoreList != null && !grrMoreList.isEmpty()) {
            snAndOperatorMap = new LinkedHashMap<>();
            for (GrrViewDataDto grrDataDto : grrMoreList) {
                String snAndOp = grrDataDto.getPart() + grrDataDto.getOperator();
                if (!snAndOperatorMap.containsKey(snAndOp)) {
                    List<GrrViewDataDto> grrDataDtoList1 = Lists.newLinkedList();
                    grrDataDtoList1.add(grrDataDto);
                    snAndOperatorMap.put(snAndOp, grrDataDtoList1);
                    grrDataDtoList1 = null;
                } else {
                    snAndOperatorMap.get(snAndOp).add(grrDataDto);
                }
            }
            for (String key : snAndOperatorMap.keySet()) {
                result.addAll(snAndOperatorMap.get(key));
            }
        }
        snAndOperatorMap = null;
        grrNormalList = null;
        grrMoreList = null;
        oldIncludeIds = null;
        oldExcludeIds = null;
        primaryMap.clear();
        changedGrrViewData = null;
        return result;
    }

    private List<GrrViewDataDto> parseGrrSlot(Map<String, List<Map<String, String>>> primaryMap, String itemNameTemp, Integer partValue, Integer operatorValue, Integer trialValue,
                                              String operatorKey, String dimension, Map<String, Set<String>> changedGrrViewData) {
        List<GrrViewDataDto> result = Lists.newLinkedList();
        List<GrrViewDataDto> grrNormalList = Lists.newLinkedList();
        List<GrrViewDataDto> grrMoreList = Lists.newLinkedList();
        Map<String, List<GrrViewDataDto>> groupMap = new LinkedHashMap<>();

        Set<String> oldIncludeIds = new HashSet<>();
        Set<String> oldExcludeIds = new HashSet<>();
        //get cache ids
        if (changedGrrViewData != null && !changedGrrViewData.isEmpty()) {
            oldIncludeIds = changedGrrViewData.get("includeIds");
            oldExcludeIds = changedGrrViewData.get("excludeIds");
        }
        boolean isMore = false;
        Integer total = operatorValue * trialValue * partValue;
        //process data
        for (String key : primaryMap.keySet()) {
            List<Map<String, String>> itemOfParts = primaryMap.get(key);
            String[] dimensions = dimension.split(",");
            for (int z = 0; z < dimensions.length; z++) {
                String dimension1 = dimensions[z];
                Integer trial = 1, k = 1, group = 1, totalCount = 0;
                String moreGroupKey = key + dimension1;

                for (Map<String, String> itemOfPart : itemOfParts) {
                    totalCount++;
                    String dimensionKeyValue = itemOfPart.get(operatorKey);
                    //process more data
                    if (totalCount == itemOfParts.size() && !dimension1.equals(dimensionKeyValue) && grrMoreList.size() != 0) {
                        List<GrrViewDataDto> grrTestDataDtoList1 = Lists.newLinkedList();
                        grrTestDataDtoList1.addAll(grrMoreList);
                        if (!groupMap.containsKey(moreGroupKey)) {
                            groupMap.put(moreGroupKey, grrTestDataDtoList1);
                        } else {
                            groupMap.get(moreGroupKey).addAll(grrTestDataDtoList1);
                        }
                        grrMoreList.clear();
                        grrTestDataDtoList1 = null;
                    }

                    if (dimension1.equals(dimensionKeyValue)) {
                        GrrViewDataDto grrDataDto = new GrrViewDataDto();
                        grrDataDto.setPart(key);
                        grrDataDto.setOperator(dimension1);

                        List<KeyValueDto> testItemNameList = Lists.newLinkedList();
                        List<String> list1 = new LinkedList<>(itemOfPart.keySet());
                        grrDataDto.setId(list1.get(itemOfPart.keySet().size() - 1));
                        list1 = null;
                        //get testItem data
                        for (String itemName : itemOfPart.keySet()) {
                            if (itemName.equals(operatorKey) || grrDataDto.getId().equals(itemName)) {
                                continue;
                            }
                            if (StringUtils.isNotBlank(itemNameTemp)) {
                                if (itemNameTemp.equals(itemName)) {
                                    KeyValueDto baseDtoResult = new KeyValueDto();
                                    baseDtoResult.setKey(itemName);
                                    baseDtoResult.setValue(StringUtils.formatBigDecimal(itemOfPart.get(itemName)));
                                    testItemNameList.add(baseDtoResult);
                                    baseDtoResult = null;
                                    break;
                                }
                            } else {
                                KeyValueDto baseDtoResult = new KeyValueDto();
                                baseDtoResult.setKey(itemName);
                                baseDtoResult.setValue(StringUtils.formatBigDecimal(itemOfPart.get(itemName)));
                                testItemNameList.add(baseDtoResult);
                                baseDtoResult = null;
                            }
                        }
                        //replace related data
                        if (oldExcludeIds.contains(grrDataDto.getId())) {
                            grrDataDto.setTrial("");
                            isMore = true;
                        } else {
                            grrDataDto.setTrial(trial.toString());
                            if (k - trialValue > 0) {
                                if (!oldIncludeIds.contains(grrDataDto.getId())) {
                                    grrDataDto.setTrial("");
                                    isMore = true;
                                }
                            }
                        }
                        k++;
                        grrDataDto.setColValueDtoList(testItemNameList);
                        testItemNameList = null;
                        //check need data
                        if (grrNormalList.size() >= total) {
                            grrDataDto.setTrial("");
                            grrDataDto.setCheck(false);
                            grrMoreList.add(grrDataDto);
                            isMore = false;
                            List<GrrViewDataDto> grrTestDataDtoList1 = Lists.newLinkedList();
                            if (grrMoreList.size() - trialValue == 0) {
                                grrTestDataDtoList1.addAll(grrMoreList);
                                if (!groupMap.containsKey(String.valueOf(group))) {
                                    groupMap.put(String.valueOf(group), grrTestDataDtoList1);
                                } else {
                                    groupMap.get(String.valueOf(group)).addAll(grrTestDataDtoList1);
                                }
                                group++;
                                grrMoreList.clear();
                            } else {
                                if (totalCount == itemOfParts.size() && (grrMoreList.size() - trialValue != 0)) {
                                    grrTestDataDtoList1.addAll(grrMoreList);
                                    if (!groupMap.containsKey(moreGroupKey)) {
                                        groupMap.put(moreGroupKey, grrTestDataDtoList1);
                                    } else {
                                        groupMap.get(moreGroupKey).addAll(grrTestDataDtoList1);
                                    }
                                    grrMoreList.clear();
                                }
                            }
                            grrTestDataDtoList1 = null;
                        } else {
                            if (isMore) {
                                grrDataDto.setCheck(false);
                                grrMoreList.add(grrDataDto);
                                isMore = false;
                                List<GrrViewDataDto> grrTestDataDtoList1 = Lists.newLinkedList();
                                if (grrMoreList.size() - trialValue == 0) {
                                    grrTestDataDtoList1.addAll(grrMoreList);
                                    if (!groupMap.containsKey(String.valueOf(group))) {
                                        groupMap.put(String.valueOf(group), grrTestDataDtoList1);
                                    } else {
                                        groupMap.get(String.valueOf(group)).addAll(grrTestDataDtoList1);
                                    }
                                    group++;
                                    grrMoreList.clear();
                                } else {
                                    if (totalCount == itemOfParts.size() && (grrMoreList.size() - trialValue != 0)) {
                                        grrTestDataDtoList1.addAll(grrMoreList);
                                        if (!groupMap.containsKey(moreGroupKey)) {
                                            groupMap.put(moreGroupKey, grrTestDataDtoList1);
                                        } else {
                                            groupMap.get(moreGroupKey).addAll(grrTestDataDtoList1);
                                        }
                                        grrMoreList.clear();
                                    }
                                }
                                grrTestDataDtoList1 = null;
                            } else {
                                trial++;
                                grrDataDto.setCheck(true);
                                grrNormalList.add(grrDataDto);
                            }
                        }
                    }
                }
            }
            itemOfParts = null;
        }
        result.addAll(grrNormalList);
        if (groupMap != null && !groupMap.isEmpty()) {
            for (String group : groupMap.keySet()) {
                grrMoreList.addAll(groupMap.get(group));
            }
        }
        result.addAll(grrMoreList);
        grrNormalList = null;
        grrMoreList = null;
        groupMap = null;
        oldIncludeIds = null;
        oldExcludeIds = null;
        primaryMap = null;
        changedGrrViewData = null;
        return result;
    }

    private GrrResultDto getSubGRRResult(GrrParamDto grrParamDto, List<GrrViewDataDto> grrViewDataDtos) {
        if (grrParamDto == null || StringUtils.isBlank(grrParamDto.getItemName()) || StringUtils.isBlank(grrParamDto.getGrrName())) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PARAMETER_INVALID));
        }
        GrrResultDto resultDto = new GrrResultDto();
        if (grrViewDataDtos == null || grrViewDataDtos.isEmpty()) {
            return resultDto;
        }
        String itemName = grrParamDto.getItemName();
        String trial = null;
        Map<String, List<KeyValueDto>> opAndTriaList = new LinkedHashMap<>();
        Map<String, List<String>> partList = new LinkedHashMap<>();

        Map<String, List<KeyValueDto>> opList = new LinkedHashMap<>();
        KeyValueDto baseDtoResult = null;
        String snTemp = null;

        for (GrrViewDataDto grrDataDto : grrViewDataDtos) {
            if (!grrDataDto.getCheck()) {
                continue;
            }
            String sn = grrDataDto.getPart();
            if (StringUtils.isBlank(snTemp)) {
                snTemp = sn;
            }
            String operator = grrDataDto.getOperator();
            trial = grrDataDto.getTrial();
            String opAndTri = operator + AppConstant.GLOBAL_SEPARATOR + trial;

            if (!partList.containsKey(snTemp)) {
                List<String> groups = Lists.newLinkedList();
                groups.add(opAndTri);
                partList.put(snTemp, groups);
            } else {
                if (partList.get(snTemp).size() < (Integer.parseInt(grrParamDto.getOperatorValue()) * Integer.parseInt(grrParamDto.getTrialValue()))) {
                    partList.get(snTemp).add(opAndTri);
                }
            }
            baseDtoResult = new KeyValueDto();
            KeyValueDto baseDtoResult1 = new KeyValueDto();
            baseDtoResult.setKey(sn);
            baseDtoResult1.setKey(sn);
            for (KeyValueDto baseDto : grrDataDto.getColValueDtoList()) {
                if (baseDto.getKey().equals(itemName)) {
                    String itemValue = (String) baseDto.getValue();
                    baseDtoResult.setValue(itemValue);
                    if (!opAndTriaList.containsKey(opAndTri)) {
                        List<KeyValueDto> colValueDtoList = Lists.newLinkedList();
                        colValueDtoList.add(baseDtoResult);
                        opAndTriaList.put(opAndTri, colValueDtoList);
                    } else {
                        List<KeyValueDto> values = opAndTriaList.get(opAndTri);
                        boolean isRepeat = false;
                        for (KeyValueDto keyValueDto : values) {
                            if (keyValueDto.getKey().equals(sn)) {
                                isRepeat = true;
                                break;
                            }
                        }
                        if (!isRepeat) {
                            opAndTriaList.get(opAndTri).add(baseDtoResult);
                        } else {
                            continue;
                        }
                    }

                    if (StringUtils.isBlankWithSpecialNumber(itemValue)) {
                        itemValue = AppConstant.GLOBAL_NAN;
                    }
                    baseDtoResult1.setValue(itemValue);
                    if (!opList.containsKey(operator)) {
                        List<KeyValueDto> itemValueList = Lists.newLinkedList();
                        itemValueList.add(baseDtoResult1);
                        opList.put(operator, itemValueList);
                    } else {
                        opList.get(operator).add(baseDtoResult1);
                    }
                    break;
                }
            }
        }
        //sub data
        List<GrrViewDataDto> grrDataDtoResultList = Lists.newLinkedList();
        List<GrrStatisticDataDto> grrStatisticDataDtoList = Lists.newLinkedList();
        Map<String, List<Double>> allSnList = new LinkedHashMap<>();
        if (partList != null && !partList.isEmpty()) {
            int count = 1;
            for (String key : partList.keySet()) {
                List<String> groupKeys = partList.get(key);
                for (String groupKey : groupKeys) {
                    String operator = groupKey.split(AppConstant.GLOBAL_SEPARATOR)[0];
                    trial = groupKey.split(AppConstant.GLOBAL_SEPARATOR)[1];
                    GrrViewDataDto grrDataDto = new GrrViewDataDto();
                    grrDataDto.setOperator(operator);
                    grrDataDto.setTrial(trial);
                    grrDataDto.setColValueDtoList(opAndTriaList.get(groupKey));
                    grrDataDtoResultList.add(grrDataDto);
                    if (count == Integer.parseInt(grrParamDto.getTrialValue())) {
                        count = 1;
                        List<KeyValueDto> itemValueList = opList.get(operator);
                        List<KeyValueDto> meanList = Lists.newLinkedList();
                        List<KeyValueDto> rangList = Lists.newLinkedList();
                        Map<String, List<Double>> snList = new LinkedHashMap<>();
                        for (int i = 0; i < itemValueList.size(); i++) {
                            String sn = itemValueList.get(i).getKey();
                            Double itemValue = Double.valueOf((String) itemValueList.get(i).getValue());
                            if (!snList.containsKey(sn)) {
                                List<Double> itemValueLists = Lists.newLinkedList();
                                itemValueLists.add(itemValue);
                                snList.put(sn, itemValueLists);
                            } else {
                                snList.get(sn).add(itemValue);
                            }
                            if (!allSnList.containsKey(sn)) {
                                List<Double> itemValueLists = Lists.newLinkedList();
                                itemValueLists.add(itemValue);
                                allSnList.put(sn, itemValueLists);
                            } else {
                                allSnList.get(sn).add(itemValue);
                            }
                            //every operator avg and rang
                            if (i == itemValueList.size() - 1) {
                                double sum = 0;
                                for (String snKey : snList.keySet()) {
                                    KeyValueDto baseDtoResult1 = new KeyValueDto();
                                    KeyValueDto baseDtoResult2 = new KeyValueDto();

                                    baseDtoResult1.setKey(snKey);
                                    baseDtoResult2.setKey(snKey);

                                    List<Double> snItemValueList = snList.get(snKey);
                                    List<Double> snItemNormalList = Lists.newArrayList();
                                    double[] itemValues = new double[snItemValueList.size()];
                                    for (int j = 0; j < snItemValueList.size(); j++) {
                                        itemValues[j] = snItemValueList.get(j);
                                        if (!StringUtils.isBlankWithSpecialNumber(String.valueOf(itemValues[j]))) {
                                            snItemNormalList.add(itemValues[j]);
                                            sum += snItemValueList.get(j);
                                        }
                                    }
                                    baseDtoResult1.setValue(getAverage(itemValues));
                                    meanList.add(baseDtoResult1);
                                    if (snItemNormalList.isEmpty()) {
                                        baseDtoResult2.setValue("-");
                                    } else {
                                        baseDtoResult2.setValue(getRange(Collections.max(snItemNormalList), Collections.min(snItemNormalList)));
                                    }
                                    rangList.add(baseDtoResult2);
                                }
                                double[] resultValues = new double[meanList.size()];
                                for (int j = 0; j < meanList.size(); j++) {
                                    if (("-").equals((String) meanList.get(j).getValue())) {
                                        resultValues[j] = Double.valueOf(AppConstant.GLOBAL_NAN);
                                    } else {
                                        resultValues[j] = Double.valueOf((String) meanList.get(j).getValue());
                                    }
                                }
                                String mean = getAverage(resultValues);
                                String rang = StringUtils.formatBigDecimal(Double.toString(Math.pow(sum, 2)));

                                GrrStatisticDataDto grrStatisticDataDto = new GrrStatisticDataDto();
                                grrStatisticDataDto.setMean(mean);
                                grrStatisticDataDto.setMeanList(meanList);
                                grrStatisticDataDto.setRangList(rangList);
                                grrStatisticDataDto.setRang(rang);
                                grrStatisticDataDtoList.add(grrStatisticDataDto);
                                resultValues = null;
                                mean = null;
                                rang = null;
                                grrStatisticDataDto = null;
                            }
                        }
                        meanList = null;
                        rangList = null;
                        itemValueList = null;
                        snList = null;
                    } else {
                        count++;
                    }
                }
            }
        }

        //total range and avg
        GrrStatisticDataDto grrStatisticDataDto = new GrrStatisticDataDto();
        List<KeyValueDto> toatlMeanList = Lists.newLinkedList();
        List<KeyValueDto> toalRangList = Lists.newLinkedList();
        for (String snKey : allSnList.keySet()) {
            KeyValueDto baseDtoResult1 = new KeyValueDto();
            KeyValueDto baseDtoResult2 = new KeyValueDto();

            baseDtoResult1.setKey(snKey);
            baseDtoResult2.setKey(snKey);

            List<Double> allSnItemValueList = allSnList.get(snKey);
            List<Double> snItemNormalList = Lists.newArrayList();
            double[] itemValues = new double[Integer.valueOf(allSnItemValueList.size())];
            for (int j = 0; j < allSnItemValueList.size(); j++) {
                itemValues[j] = allSnItemValueList.get(j);
                if (!StringUtils.isBlankWithSpecialNumber(String.valueOf(itemValues[j]))) {
                    snItemNormalList.add(itemValues[j]);
                }
            }
            baseDtoResult1.setValue(getAverage(itemValues));

            toatlMeanList.add(baseDtoResult1);
            if (snItemNormalList.isEmpty()) {
                baseDtoResult2.setValue("-");
            } else {
                baseDtoResult2.setValue(getRange(Collections.max(snItemNormalList), Collections.min(snItemNormalList)));
            }
            toalRangList.add(baseDtoResult2);

            baseDtoResult1 = null;
            baseDtoResult2 = null;
            allSnItemValueList = null;
            snItemNormalList = null;
        }
        grrStatisticDataDto.setTotalMeanList(toatlMeanList);
        grrStatisticDataDto.setTotalRangList(toalRangList);
        grrStatisticDataDtoList.add(grrStatisticDataDto);

        resultDto.setGrrViewDataDtoList(grrDataDtoResultList);
        resultDto.setGrrStatisticDataDtoList(grrStatisticDataDtoList);

        opAndTriaList = null;
        partList = null;
        opList = null;
        toatlMeanList = null;
        toalRangList = null;
        grrStatisticDataDto = null;
        grrDataDtoResultList = null;
        grrStatisticDataDtoList = null;
        return resultDto;
    }

    private String getAverage(double[] array) {
        double sum = 0, count = 0;
        if (array == null || array.length == 0) {
            return "-";
        }
        for (int i = 0; i < array.length; i++) {
            if (StringUtils.isBlankWithSpecialNumber(String.valueOf(array[i]))) {
                continue;
            }
            sum += array[i];
            count++;
        }
        if (count == 0) {
            return "-";
        }
        return StringUtils.formatBigDecimal(String.valueOf(sum / count));
    }

    private String getRange(double max, double min) {
        return StringUtils.formatBigDecimal(String.valueOf(max - min));
    }

    private void validateTestDatas(String filePath, Map<String, List<String>> testDatas, GrrConfigDto grrConfigDto, GrrItemDto grrItemDto, List<String> groupResultValues) {
        if (StringUtils.isBlank(filePath) || testDatas == null || testDatas.isEmpty() || grrConfigDto == null) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PARAMETER_INVALID));
        }

        String partKey = grrConfigDto.getPartKey();
        String scheme = grrConfigDto.getScheme();
        String operatorKey = grrConfigDto.getOperatorKey();

        String modelValue = grrConfigDto.getModelValue();
        String[] modelValues = modelValue.substring(1, modelValue.length() - 1).split(",");

        Integer partValue = Integer.valueOf(modelValues[0]);
        Integer operatorValue = Integer.valueOf(modelValues[1]);
        Integer trialValue = Integer.valueOf(modelValues[2]);
        modelValues = null;
        Integer partNumber = operatorValue * trialValue;
        String dimension = grrItemDto.getDimension();

        Map<String, Integer> analysisKeysMap = new LinkedHashMap<>();
        Map<String, Integer> groupKeysMap = new LinkedHashMap<>();

        Set<String> groupValues = new LinkedHashSet<>();
        Set<String> operatorValues = new LinkedHashSet<>();

        List<String> partList = Lists.newLinkedList();
        List<String> operatorValueList = Lists.newLinkedList();

        String partKeyValue = null;
        String group1KeyValue = null;
        String group2KeyValue = null;
        String operatorKeyValue = null;
        if (testDatas != null && !testDatas.isEmpty()) {
            for (String mapKey : testDatas.keySet()) {
                if (mapKey.contains(AppConstant.DATA_INFO)) {
                    List<Map> dataList = mapper.fromJson(testDatas.get(mapKey).toString(), List.class);
                    for (int i = 0; i < dataList.size(); i++) {
                        Map<String, String> row = dataList.get(i);
                        partKeyValue = row.get(partKey);
                        if (StringUtils.isBlank(partKeyValue)) {
                            continue;
                        }
                        group1KeyValue = row.get(grrConfigDto.getGroup1Key());
                        group2KeyValue = row.get(grrConfigDto.getGroup2Key());
                        operatorKeyValue = row.get(operatorKey);

                        String analysisKeys = null;
                        String groupKeys = null;
                        String groupValue = null;
                        if (StringUtils.isNotBlank(group2KeyValue) && StringUtils.isNotBlank(operatorKeyValue)) {
                            analysisKeys = partKeyValue + AppConstant.GLOBAL_SEPARATOR + group1KeyValue + AppConstant.GLOBAL_SEPARATOR + group2KeyValue + AppConstant.GLOBAL_SEPARATOR + operatorKeyValue;
                            groupKeys = partKeyValue + AppConstant.GLOBAL_SEPARATOR + group1KeyValue + AppConstant.GLOBAL_SEPARATOR + group2KeyValue;
                            groupValue = group1KeyValue + AppConstant.GLOBAL_SEPARATOR + group2KeyValue;
                            if (operatorValues.size() < operatorValue) {
                                operatorValues.add(operatorKeyValue);
                            }
                        } else if (StringUtils.isNotBlank(group2KeyValue) && StringUtils.isBlank(operatorKeyValue)) {
                            analysisKeys = partKeyValue + AppConstant.GLOBAL_SEPARATOR + group1KeyValue + AppConstant.GLOBAL_SEPARATOR + group2KeyValue;
                        } else if (StringUtils.isBlank(group2KeyValue) && StringUtils.isNotBlank(operatorKeyValue)) {
                            analysisKeys = partKeyValue + AppConstant.GLOBAL_SEPARATOR + group1KeyValue + AppConstant.GLOBAL_SEPARATOR + operatorKeyValue;
                            groupValue = group1KeyValue;
                            groupKeys = partKeyValue + AppConstant.GLOBAL_SEPARATOR + group1KeyValue;
                            if (operatorValues.size() < operatorValue) {
                                operatorValues.add(operatorKeyValue);
                            }
                        } else {
                            analysisKeys = partKeyValue + AppConstant.GLOBAL_SEPARATOR + group1KeyValue;
                        }
                        groupValues.add(groupValue);
                        if (!partList.contains(partKeyValue)) {
                            partList.add(partKeyValue);
                        }

                        if (!operatorValueList.contains(operatorKeyValue)) {
                            operatorValueList.add(operatorKeyValue);
                        }

                        if (!analysisKeysMap.containsKey(analysisKeys)) {
                            analysisKeysMap.put(analysisKeys, 1);
                        } else {
                            analysisKeysMap.put(analysisKeys, analysisKeysMap.get(analysisKeys) + 1);
                        }

                        if (!groupKeysMap.containsKey(groupKeys)) {
                            groupKeysMap.put(groupKeys, 1);
                        } else {
                            groupKeysMap.put(groupKeys, groupKeysMap.get(groupKeys) + 1);
                        }
                        row = null;
                    }
                    dataList = null;
                }
            }
        }

        if (partList == null || partList.isEmpty() || partList.size() < partValue) {
            logger.error("Lack of Part: [{}]", (partValue - partList.size()));
            String[] args = new String[]{(String.valueOf(partValue - partList.size()))};
            throw new ApplicationException(ExceptionMessages.ERR_12004, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PART_COUNT_INVALID, args));
        }

        List<String> errorAnalysisKeys = Lists.newLinkedList();
        List<String> errorDimensionValues = Lists.newLinkedList();
        if (AppConstant.GRR_MODEL_SLOT.equals(scheme)) {
            String[] testStrategy = null;
            if (StringUtils.isNotBlank(dimension)) {
                testStrategy = dimension.split(";");
                if (testStrategy != null && testStrategy.length != 0) {
                    Map<String, Integer> groupKeysMap1 = new LinkedHashMap<>();
                    for (String ts : testStrategy) {
                        String[] dimensionValues = ts.split(",");
                        if (operatorValue != dimensionValues.length) {
                            logger.error("Operator does not match dimension");
                            throw new ApplicationException(ExceptionMessages.ERR_12007, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_OPERATOR_DIMENSION_NOT_MATCH));
                        }
                        for (int i = 0; i < dimensionValues.length; i++) {
                            if (!operatorValueList.contains(dimensionValues[i])) {
                                errorDimensionValues.add(dimensionValues[i]);
                            }
                            for (String key : groupKeysMap.keySet()) {
                                key = key + AppConstant.GLOBAL_SEPARATOR + dimensionValues[i];
                                if (!groupKeysMap1.containsKey(key)) {
                                    groupKeysMap1.put(key, 1);
                                } else {
                                    groupKeysMap1.put(key, groupKeysMap1.get(key) + 1);
                                }
                            }
                        }
                        dimensionValues = null;
                    }
                    testStrategy = null;
                    if (!errorDimensionValues.isEmpty()) {
                        logger.error("Lack of Operator: [{}]", errorDimensionValues);
                        String[] args = new String[]{errorDimensionValues.toString()};
                        throw new ApplicationException(ExceptionMessages.ERR_12008, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_OPERATOR_STRATEGY_INVALID, args));
                    }

                    int lackOfTotal = 0;
                    for (String key : groupKeysMap1.keySet()) {
                        if (!analysisKeysMap.keySet().contains(key)) {
                            lackOfTotal++;
                        }
                        if (analysisKeysMap.get(key) == null || analysisKeysMap.get(key) < trialValue) {
                            errorAnalysisKeys.add(key.replace(AppConstant.GLOBAL_SEPARATOR, "_"));
                        }
                    }

                    if (lackOfTotal != 0) {
                        logger.error("Lack of Group: [{}]", lackOfTotal);
                        Integer[] args = new Integer[lackOfTotal];
                        throw new ApplicationException(ExceptionMessages.ERR_12006, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_GROUP_INVALID, args));
                    }

                    if (!errorAnalysisKeys.isEmpty()) {
                        logger.error("Part number error: [{}]", errorAnalysisKeys);
                        String[] args = new String[]{errorAnalysisKeys.toString()};
                        throw new ApplicationException(ExceptionMessages.ERR_12005, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_DATA_COUNT_INVALID, args));
                    }
                    groupKeysMap1 = null;
                    groupValues = null;
                }
            } else {
                if (operatorValues != null && !operatorValues.isEmpty()) {
                    Map<String, Integer> groupKeysMap1 = new LinkedHashMap<>();
                    String dimensionResult = "";
                    if (operatorValues.size() != operatorValue) {
                        logger.error("The operator setting is not matching with the number of group in source file.");
                        String[] args = new String[]{String.valueOf(operatorValue - operatorValues.size())};
                        throw new ApplicationException(ExceptionMessages.ERR_12012, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_OPERATOR_NUMBER_ERROR, args));
                    }
                    for (String ts : operatorValues) {
                        dimensionResult += ts + ",";
                        if (!operatorValueList.contains(ts)) {
                            errorDimensionValues.add(ts);
                        }
                        for (String key : groupKeysMap.keySet()) {
                            key = key + AppConstant.GLOBAL_SEPARATOR + ts;
                            if (!groupKeysMap1.containsKey(key)) {
                                groupKeysMap1.put(key, 1);
                            } else {
                                groupKeysMap1.put(key, groupKeysMap1.get(key) + 1);
                            }
                        }
                    }
                    if (!errorDimensionValues.isEmpty()) {
                        logger.error("Lack of Operator: [{}]", errorDimensionValues);
                        String[] args = new String[]{errorDimensionValues.toString()};
                        throw new ApplicationException(ExceptionMessages.ERR_12008, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_OPERATOR_STRATEGY_INVALID, args));
                    }
                    int lackOfTotal = 0;
                    for (String key : groupKeysMap1.keySet()) {
                        if (!analysisKeysMap.keySet().contains(key)) {
                            lackOfTotal++;
                        }
                        if (analysisKeysMap.get(key) == null || analysisKeysMap.get(key) < trialValue) {
                            errorAnalysisKeys.add(key.replace(AppConstant.GLOBAL_SEPARATOR, "_"));
                        }
                    }
                    if (lackOfTotal != 0) {
                        logger.error("Lack of Group: [{}]", lackOfTotal);
                        Integer[] args = new Integer[lackOfTotal];
                        throw new ApplicationException(ExceptionMessages.ERR_12006, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_GROUP_INVALID, args));
                    }

                    if (!errorAnalysisKeys.isEmpty()) {
                        logger.error("Part number error: [{}]", errorAnalysisKeys);
                        String[] args = new String[]{errorAnalysisKeys.toString()};
                        throw new ApplicationException(ExceptionMessages.ERR_12005, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_DATA_COUNT_INVALID, args));
                    }

                    for (String key : groupValues) {
                        groupResultValues.add(key + AppConstant.GLOBAL_SEPARATOR + dimensionResult.substring(0, dimensionResult.length() - 1));
                        grrItemDto.setDimension(dimensionResult.substring(0, dimensionResult.length() - 1));
                    }
                    groupKeysMap1 = null;
                    groupValues = null;
                } else {
                    String[] args = new String[]{operatorKey};
                    throw new ApplicationException(ExceptionMessages.ERR_12011, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_OPERATOR_IS_EMPTY, args));
                }
            }
        } else {
            for (String key : analysisKeysMap.keySet()) {
                if (analysisKeysMap.get(key) == null || analysisKeysMap.get(key) < partNumber) {
                    errorAnalysisKeys.add(key.replace(AppConstant.GLOBAL_SEPARATOR, "_"));
                }
            }
            if (!errorAnalysisKeys.isEmpty()) {
                logger.error("Part number error: [{}]", errorAnalysisKeys);
                String[] args = new String[]{errorAnalysisKeys.toString()};
                throw new ApplicationException(ExceptionMessages.ERR_12005, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_DATA_COUNT_INVALID, args));
            }
        }
        analysisKeysMap = null;
        groupKeysMap = null;
        partList = null;
        operatorValueList = null;
        errorAnalysisKeys = null;
        errorDimensionValues = null;
    }

    private void validateParams(GrrConfigDto grrConfigDto, GrrItemDto grrItemDto, GrrParamDto grrParamDto, SystemConfigDto systemConfigDto) {
        if (grrConfigDto == null || grrItemDto == null) {
            throw new ApplicationException(ExceptionMessages.ERR_12003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_CONFIG_INVALID));
        }
        String partKey = grrConfigDto.getPartKey();
        String operatorKey = grrConfigDto.getOperatorKey();
        String group1Key = grrConfigDto.getGroup1Key();

        String scheme = grrConfigDto.getScheme();
        String firstChange = grrConfigDto.getFirstChange();
        String sigma = grrConfigDto.getSigma();

        String dimension = grrItemDto.getDimension();

        if (StringUtils.isBlank(scheme) || StringUtils.isBlank(partKey) || StringUtils.isBlank(group1Key) || StringUtils.isBlank(sigma)) {
            logger.error("scheme or partKey, group1key, sigma is empty.");
            throw new ApplicationException(ExceptionMessages.ERR_12003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_CONFIG_INVALID));
        }

        String modelValue = grrConfigDto.getModelValue();
        if (StringUtils.isBlank(modelValue)) {
            logger.error("model is empty.");
            throw new ApplicationException(ExceptionMessages.ERR_12003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_CONFIG_INVALID));
        }

        String[] modelValues = modelValue.substring(1, modelValue.length() - 1).split(",");
        final int length = 3;
        if (modelValues == null || modelValues.length != length) {
            logger.error("model format error.");
            throw new ApplicationException(ExceptionMessages.ERR_12003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_CONFIG_INVALID));
        }
        Integer partValue = Integer.valueOf(modelValues[0]);
        Integer operatorValue = Integer.valueOf(modelValues[1]);
        Integer trialValue = Integer.valueOf(modelValues[2]);
        modelValues = null;

        if (AppConstant.GRR_MODEL_NORMAL.equals(scheme)) {
            if (StringUtils.isBlank(firstChange)) {
                logger.error("firstChange is empty.");
                throw new ApplicationException(ExceptionMessages.ERR_12003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_CONFIG_INVALID));
            }
        } else if (scheme.equals(AppConstant.GRR_MODEL_SLOT)) {
            if (StringUtils.isBlank(operatorKey)) {
                logger.error("operatorKey is empty.");
                throw new ApplicationException(ExceptionMessages.ERR_12003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_CONFIG_INVALID));
            }
            if (StringUtils.isBlank(dimension)) {
                logger.error("Test strategy is empty.");
            }
        }

        if (grrParamDto != null) {
            String fileName = grrParamDto.getFilePath().substring(grrParamDto.getFilePath().lastIndexOf(File.separator) + 1, grrParamDto.getFilePath().length());
            //refresh sigma
            if (StringUtils.isBlank(grrParamDto.getSigma())) {
                grrParamDto.setSigma(grrConfigDto.getSigma());
            }
            grrParamDto.setGrrName(fileName);
            grrParamDto.setOperatorValue(operatorValue.toString());
            grrParamDto.setPartValue(partValue.toString());
            grrParamDto.setTrialValue(trialValue.toString());
            grrParamDto.setRule(grrConfigDto.getRule());
        }

        if (systemConfigDto != null && grrParamDto != null) {
            grrParamDto.setDigitNum(grrConfigDto.getDigit());
        }
    }

    private GrrResultDto searchGrrResultForExport(GrrParamDto grrParamDto, List<GrrViewDataDto> grrViewDataDtos, Map<String, Object> uslAndLslAndUnits) {
        GrrResultDto resultDto = this.getSubGRRResult(grrParamDto, grrViewDataDtos);
        GrrAnalysisDataDto grrAnalysisDataDto = this.getGrrAnalysisData(grrParamDto, grrViewDataDtos, uslAndLslAndUnits);
        Map<String, Object> grrAnovaDtoAndSourceDto = analysisService.analyseGrrAnovaAndSource(grrParamDto, grrAnalysisDataDto);
        List<GrrAnovaDto> grrAnovaDtoList = (List<GrrAnovaDto>) grrAnovaDtoAndSourceDto.get(AppConstant.GRR_ANOVA_VALUE);
        List<GrrSourceDto> grrSourceDtoList = (List<GrrSourceDto>) grrAnovaDtoAndSourceDto.get(AppConstant.GRR_SOURCE_VALUE);

        GrrChartDto grrChartDto = analysisService.analyseGrrChartData(grrParamDto, grrAnalysisDataDto);
        resultDto.setGrrAnovaDtoList(grrAnovaDtoList);
        resultDto.setGrrSourceDtoList(grrSourceDtoList);
        resultDto.setGrrChartDto(grrChartDto);
        grrChartDto = null;
        grrSourceDtoList = null;
        grrAnovaDtoList = null;
        grrParamDto = null;
        grrViewDataDtos = null;

        return resultDto;
    }

    private List<GrrAnalysisDataDto> getGrrAnalysisDatas(GrrParamDto grrParamDto, List<GrrViewDataDto> grrViewDataDtos, Map<String, Object> uslAndLslAndUnits) {
        if (grrParamDto == null || StringUtils.isBlank(grrParamDto.getFilePath())) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PARAMETER_INVALID));
        }
        List<GrrAnalysisDataDto> grrAnalysisParamDtoList = Lists.newLinkedList();
        if (grrViewDataDtos == null || grrViewDataDtos.isEmpty()) {
            return grrAnalysisParamDtoList;
        }

        Map<String, List<String>> itemNameList = new LinkedHashMap<>();
        for (GrrViewDataDto grrDataDto : grrViewDataDtos) {
            if (!grrDataDto.getCheck()) {
                continue;
            }
            for (KeyValueDto baseDto : grrDataDto.getColValueDtoList()) {
                String itemName = baseDto.getKey();
                String itemValue = (String) baseDto.getValue();
                if (StringUtils.isBlankWithSpecialNumber(itemValue)) {
                    itemValue = AppConstant.GLOBAL_NAN;
                }
                if (!itemNameList.containsKey(itemName)) {
                    List<String> itemValueList = Lists.newLinkedList();
                    itemValueList.add(itemValue);
                    itemNameList.put(itemName, itemValueList);
                    itemValueList = null;
                } else {
                    itemNameList.get(itemName).add(itemValue);
                }
            }
        }

        //refresh usl and lsl
        Double uslResult = Double.valueOf(AppConstant.GLOBAL_NAN);
        Double lslResult = Double.valueOf(AppConstant.GLOBAL_NAN);

        if (StringUtils.isNotBlank(grrParamDto.getUsl()) && StringUtils.isNumeric(grrParamDto.getUsl())) {
            uslResult = Double.valueOf(grrParamDto.getUsl());
        }

        if (StringUtils.isNotBlank(grrParamDto.getLsl()) && StringUtils.isNumeric(grrParamDto.getLsl())) {
            lslResult = Double.valueOf(grrParamDto.getLsl());
        }

        for (String key : itemNameList.keySet()) {
            List<String> itemValueList = itemNameList.get(key);
            boolean invalidate = false;
            double[] itemValues = new double[itemValueList.size()];
            for (int i = 0; i < itemValueList.size(); i++) {
                String itemValue = itemValueList.get(i);
                if (!StringUtils.isBlankWithSpecialNumber(itemValue) && !StringUtils.isNumeric(itemValue)) {
                    itemValue = AppConstant.GLOBAL_NAN;
                    invalidate = true;
                }
                if (StringUtils.isBlankWithSpecialNumber(itemValue)) {
                    itemValue = AppConstant.GLOBAL_NAN;
                }
                itemValues[i] = Double.valueOf(itemValue);
            }
            itemValueList = null;
            GrrAnalysisDataDto grrAnalysisParamDto = new GrrAnalysisDataDto();
            if (uslAndLslAndUnits != null && !uslAndLslAndUnits.isEmpty()) {
                grrAnalysisParamDto.setItemName(key);
                grrAnalysisParamDto.setInvalidate(invalidate);
                grrAnalysisParamDto.setAnalysisData(itemValues);
                String param = (String) uslAndLslAndUnits.get(key);
                String[] paramValue = param == null ? null : param.split(",", -1);
                final int length = 3;
                if (uslResult.isNaN() && lslResult.isNaN()) {
                    if (StringUtils.isBlank(param) || paramValue == null || paramValue.length != length) {
                        grrAnalysisParamDto.setUsl(Double.valueOf(AppConstant.GLOBAL_NAN));
                        grrAnalysisParamDto.setLsl(Double.valueOf(AppConstant.GLOBAL_NAN));
                    } else {
                        grrAnalysisParamDto.setUsl(Double.valueOf(StringUtils.isNumeric(paramValue[0]) ? paramValue[0] : AppConstant.GLOBAL_NAN));
                        grrAnalysisParamDto.setLsl(Double.valueOf(StringUtils.isNumeric(paramValue[1]) ? paramValue[1] : AppConstant.GLOBAL_NAN));
                    }
                } else {
                    grrAnalysisParamDto.setUsl(uslResult.doubleValue());
                    grrAnalysisParamDto.setLsl(lslResult.doubleValue());
                }

                grrAnalysisParamDtoList.add(grrAnalysisParamDto);
            }
        }
        itemNameList = null;
        return grrAnalysisParamDtoList;
    }

    private GrrAnalysisDataDto getGrrAnalysisData(GrrParamDto grrParamDto, List<GrrViewDataDto> grrViewDataDtos, Map<String, Object> uslAndLslAndUnits) {
        if (grrParamDto == null || StringUtils.isBlank(grrParamDto.getFilePath()) || StringUtils.isBlank(grrParamDto.getItemName())) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PARAMETER_INVALID));
        }
        GrrAnalysisDataDto grrAnalysisParamDto = new GrrAnalysisDataDto();
        if (grrViewDataDtos == null || grrViewDataDtos.isEmpty()) {
            return grrAnalysisParamDto;
        }
        Map<String, List<String>> itemNameList = new LinkedHashMap<>();
        String itemName = grrParamDto.getItemName();
        for (GrrViewDataDto grrDataDto : grrViewDataDtos) {
            if (!grrDataDto.getCheck()) {
                continue;
            }
            for (KeyValueDto baseDto : grrDataDto.getColValueDtoList()) {
                if (itemName != null && itemName.equals(baseDto.getKey())) {
                    String itemValue = (String) baseDto.getValue();
                    if (!StringUtils.isBlankWithSpecialNumber(itemValue) && !StringUtils.isNumeric(itemValue)) {
                        itemValue = AppConstant.GLOBAL_NAN;
                    }
                    if (StringUtils.isBlankWithSpecialNumber(itemValue)) {
                        itemValue = AppConstant.GLOBAL_NAN;
                    }
                    if (!itemNameList.containsKey(itemName)) {
                        List<String> itemValueList = Lists.newLinkedList();
                        itemValueList.add(itemValue);
                        itemNameList.put(itemName, itemValueList);
                        itemValueList = null;
                    } else {
                        itemNameList.get(itemName).add(itemValue);
                    }
                }
            }
        }
        //refresh usl and lsl
        Double uslResult = Double.valueOf(AppConstant.GLOBAL_NAN);
        Double lslResult = Double.valueOf(AppConstant.GLOBAL_NAN);

        if (StringUtils.isNotBlank(grrParamDto.getUsl()) && StringUtils.isNumeric(grrParamDto.getUsl())) {
            uslResult = Double.valueOf(grrParamDto.getUsl());
        }

        if (StringUtils.isNotBlank(grrParamDto.getLsl()) && StringUtils.isNumeric(grrParamDto.getLsl())) {
            lslResult = Double.valueOf(grrParamDto.getLsl());
        }
        //only one
        for (String key : itemNameList.keySet()) {
            List<String> itemValueList = itemNameList.get(key);
            boolean invalidate = false;
            double[] itemValues = new double[itemValueList.size()];
            for (int i = 0; i < itemValueList.size(); i++) {
                String itemValue = itemValueList.get(i);
                if (!StringUtils.isBlankWithSpecialNumber(itemValue) && !StringUtils.isNumeric(itemValue)) {
                    itemValue = AppConstant.GLOBAL_NAN;
                    invalidate = true;
                }
                if (StringUtils.isBlankWithSpecialNumber(itemValue)) {
                    itemValue = AppConstant.GLOBAL_NAN;
                }
                itemValues[i] = Double.valueOf(itemValue);
            }
            itemValueList = null;
            if (uslAndLslAndUnits != null && !uslAndLslAndUnits.isEmpty()) {
                grrAnalysisParamDto.setItemName(key);
                grrAnalysisParamDto.setInvalidate(invalidate);
                grrAnalysisParamDto.setAnalysisData(itemValues);
                String param = (String) uslAndLslAndUnits.get(key);
                String[] paramValue = param == null ? null : param.split(",", -1);
                if (uslResult.isNaN() && lslResult.isNaN()) {
                    final int length = 3;
                    if (StringUtils.isBlank(param) || paramValue == null || paramValue.length != length) {
                        grrAnalysisParamDto.setUsl(Double.valueOf(AppConstant.GLOBAL_NAN));
                        grrAnalysisParamDto.setLsl(Double.valueOf(AppConstant.GLOBAL_NAN));
                    } else {
                        grrAnalysisParamDto.setUsl(Double.valueOf(StringUtils.isNumeric(paramValue[0]) ? paramValue[0] : AppConstant.GLOBAL_NAN));
                        grrAnalysisParamDto.setLsl(Double.valueOf(StringUtils.isNumeric(paramValue[1]) ? paramValue[1] : AppConstant.GLOBAL_NAN));
                    }
                } else {
                    grrAnalysisParamDto.setUsl(uslResult.doubleValue());
                    grrAnalysisParamDto.setLsl(lslResult.doubleValue());
                }
            }
        }
        itemNameList = null;
        return grrAnalysisParamDto;
    }

    private List<GrrSummaryDto> getGrrSummary(GrrParamDto grrParamDto, List<GrrViewDataDto> grrViewDataDtos, Map<String, Object> uslAndLslAndUnits) {
        if (grrParamDto == null || StringUtils.isBlank(grrParamDto.getFilePath())) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PARAMETER_INVALID));
        }
        List<GrrSummaryDto> grrSummaryList = analysisService.analyseGrrSummary(grrParamDto, this.getGrrAnalysisDatas(grrParamDto, grrViewDataDtos, uslAndLslAndUnits));
        return grrSummaryList;
    }

    private List<GrrViewDataDto> getGrrViewDatas(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey, List<String> itemNameCheckedLists, GrrConfigDto grrConfigDto, Map<String, List<String>> testDatas) {
        String filePath = grrParamDto.getFilePath();
        String partKey = grrConfigDto.getPartKey();
        String scheme = grrConfigDto.getScheme();
        String firstChange = grrConfigDto.getFirstChange();
        String operatorKey = grrConfigDto.getOperatorKey();

        String modelValue = grrConfigDto.getModelValue();
        String[] modelValues = modelValue.substring(1, modelValue.length() - 1).split(",");

        Integer partValue = Integer.valueOf(modelValues[0]);
        Integer operatorValue = Integer.valueOf(modelValues[1]);
        Integer trialValue = Integer.valueOf(modelValues[2]);
        modelValues = null;

        grrParamDto.setOperatorValue(operatorValue.toString());
        grrParamDto.setPartValue(partValue.toString());
        grrParamDto.setTrialValue(trialValue.toString());
        grrParamDto.setRule(grrConfigDto.getRule());

        Map<String, List<Map<String, String>>> primaryMap = new LinkedHashMap<>();
        if (testDatas != null && !testDatas.isEmpty()) {
            for (String mapKey : testDatas.keySet()) {
                if (mapKey.contains(AppConstant.DATA_INFO)) {
                    String datas = testDatas.get(mapKey).toString();
                    List<Map> dataList = mapper.fromJson(datas, List.class);
                    for (int i = 0; i < dataList.size(); i++) {
                        Map<String, String> row = dataList.get(i);
                        Map<String, String> rowResult = new LinkedHashMap<>();
                        if (StringUtils.isNotBlank(groupSearchKey.getGroupkey1())) {
                            if (!groupSearchKey.getGroupValue1().equals(row.get(grrConfigDto.getGroup1Key()))) {
                                continue;
                            }
                        }
                        if (StringUtils.isNotBlank(groupSearchKey.getGroupkey2())) {
                            if (!groupSearchKey.getGroupValue2().equals(row.get(grrConfigDto.getGroup2Key()))) {
                                continue;
                            }
                        }
                        if (StringUtils.isNotBlank(operatorKey) && StringUtils.isNotBlank(groupSearchKey.getDimensionValue())) {
                            if (!groupSearchKey.getDimensionValue().contains(row.get(operatorKey))) {
                                continue;
                            }
                        }
                        if (StringUtils.isNotBlank(partKey)) {
                            String primaryValue = row.get(partKey);
                            if (!StringUtils.isBlank(primaryValue)) {
                                if (!primaryMap.containsKey(primaryValue)) {
                                    List<Map<String, String>> items = Lists.newArrayList();
                                    for (String s : row.keySet()) {
                                        if (itemNameCheckedLists.contains(s)) {
                                            rowResult.put(s, row.get(s));
                                        }
                                    }
                                    if (StringUtils.isNotBlank(operatorKey)) {
                                        rowResult.put(operatorKey, row.get(operatorKey));
                                    }
                                    rowResult.put(primaryValue + 1, primaryValue + 1);
                                    items.add(rowResult);
                                    primaryMap.put(primaryValue, items);
                                    items = null;
                                } else {
                                    for (String s : row.keySet()) {
                                        if (itemNameCheckedLists.contains(s)) {
                                            rowResult.put(s, row.get(s));
                                        }
                                    }
                                    if (StringUtils.isNotBlank(operatorKey)) {
                                        rowResult.put(operatorKey, row.get(operatorKey));
                                    }
                                    long count = primaryMap.get(primaryValue).size();
                                    count++;
                                    rowResult.put(primaryValue + count, primaryValue + count);
                                    primaryMap.get(primaryValue).add(rowResult);
                                }
                            }
                        }
                        row = null;
                        rowResult = null;
                    }
                    dataList = null;
                }
            }
        }
        String itemName = grrParamDto.getItemName();
        Map<String, Set<String>> changedGrrViewData = null;
        try {
            String grrViewData = grrDataCacheFactory.getCacheGrrViewData().get(filePath + groupSearchKey);
            if (StringUtils.isNotBlank(grrViewData) && !grrViewData.equals("[]")) {
                changedGrrViewData = mapper.fromJson(grrViewData, mapper.contructMapType(Map.class, String.class, Set.class));
            }
        } catch (ExecutionException e) {
            logger.error("grr view datas error");
            throw new ApplicationException(ExceptionMessages.ERR_12003, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_CONFIG_INVALID));
        }
        testDatas = null;
        if (AppConstant.GRR_MODEL_NORMAL.equals(scheme) && AppConstant.GRR_FIRST_CHANGE_OPERATOR.equals(firstChange)) {
            return this.parseGrrNormalForOperator(primaryMap, itemName, partValue, operatorValue, trialValue, changedGrrViewData);
        } else if (AppConstant.GRR_MODEL_NORMAL.equals(scheme) && AppConstant.GRR_FIRST_CHANGE_TRIAL.equals(firstChange)) {
            return this.parseGrrNormalForTrial(primaryMap, itemName, partValue, operatorValue, trialValue, changedGrrViewData);
        } else {
            String dimension = groupSearchKey.getDimensionValue();
            return this.parseGrrSlot(primaryMap, itemName, partValue, operatorValue, trialValue, operatorKey, dimension, changedGrrViewData);
        }
    }

    private List<String> getAllColNames(List<List<String>> colNamesObject) {
        List<String> allColNames = new ArrayList<>();
        colNamesObject.forEach(allColNames::addAll);
        return allColNames;
    }

    private Set<String> getSearchKeys(List<String> searchKeysObject, SystemConfigDto systemConfigDto) {
        Set<String> searchKeys;
        if (searchKeysObject == null) {
            searchKeys = new LinkedHashSet<>(systemConfigDto.getSearchKeyKeys());
        } else {
            searchKeys = new LinkedHashSet<>();
            List<List<String>> searchKeysList = mapper.fromJson(searchKeysObject.toString(), List.class);
            searchKeysList.forEach(searchKeys::addAll);
        }
        return searchKeys;
    }

    private Set<String> getAttributeKeys(List<String> attributeKeysObject, SystemConfigDto systemConfigDto) {
        Set<String> attributeKeys;
        if (attributeKeysObject == null) {
            attributeKeys = new LinkedHashSet<>(systemConfigDto.getSearchKeyKeys());
        } else {
            attributeKeys = new LinkedHashSet<>();
            List<List<String>> attributeKeysList = mapper.fromJson(attributeKeysObject.toString(), List.class);
            attributeKeysList.forEach(attributeKeys::addAll);
        }
        return attributeKeys;
    }

    private Set<String> getSearchAndAttributeKeys(List<String> filePaths, String scriptPath, SystemConfigDto systemConfigDto) {
        PythonUtil pythonUtil = new PythonUtil();
        List<String> searchSourceKeys = pythonUtil.getSearchKeys(filePaths, scriptPath);
        List<String> attributeSourceKeys = pythonUtil.getAttributeKeys(filePaths, scriptPath);
        Set<String> result = new LinkedHashSet<>();
        result.addAll(this.getSearchKeys(searchSourceKeys, systemConfigDto));
        result.addAll(this.getAttributeKeys(attributeSourceKeys, systemConfigDto));
        searchSourceKeys = null;
        attributeSourceKeys = null;
        return result;
    }

    private String getScriptPath() {
        SysConfigDto sysConfigDto = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_SCRIPT_PATH);
        String scriptPath = null;
        if (sysConfigDto != null && StringUtils.isNotBlank(sysConfigDto.getValue())) {
            scriptPath = sysConfigDto.getValue();
        }
        if (StringUtils.isBlank(scriptPath)) {
            scriptPath = ApplicationPathUtil.getPath(AppConstant.PATH_SCRIPT, AppConstant.PATH_SCRIPT_FILENAME);
        }
        sysConfigDto = null;
        return scriptPath;
    }

    private List<String> getTestItemKeys(List<String> itemNameObject) {
        List<List<String>> itemNameList = mapper.fromJson(itemNameObject.toString(), List.class);
        Set<String> testItemNamesSet = new LinkedHashSet<>();
        itemNameList.forEach(testItemNamesSet::addAll);
        List<String> testItemNames = new ArrayList<>(testItemNamesSet);
        itemNameList = null;
        testItemNamesSet = null;
        return testItemNames;

    }

    private Map<String, Object> getUslAndLslAndUnits(List<String> filePaths, String scriptPath) {
        PythonUtil pythonUtil = new PythonUtil();
        List<String> testDatas = pythonUtil.getParamKeys(filePaths, scriptPath);
        Map<String, Object> paramMap = new LinkedHashMap<>();
        if (testDatas != null && !testDatas.isEmpty()) {
            paramMap = mapper.fromJson(testDatas.get(0).toString(), Map.class);
        }
        return paramMap;
    }

    private Map<String, List<String>> getDataByName(List<String> filePaths, String scriptPath, List<String> itemNameCheckedLists) {
        PythonUtil pythonUtil = new PythonUtil();
        Map<String, List<String>> testDatas = pythonUtil.getDataByName(filePaths, itemNameCheckedLists, scriptPath);
        return testDatas;
    }

    private Map<String, List<String>> getDataByFile(List<String> filePaths, String scriptPath, String itemNameFilePath) {
        PythonUtil pythonUtil = new PythonUtil();
        Map<String, List<String>> testDatas = pythonUtil.getDataByFile(filePaths, scriptPath, itemNameFilePath);
        return testDatas;
    }

    private List<GrrViewDataDto> getGrrViewDataForItemName(String itemName, List<GrrViewDataDto> allGrrViewDataDtos) {
        List<GrrViewDataDto> grrViewDataDtos = Lists.newLinkedList();
        for (GrrViewDataDto grrViewDataDto : allGrrViewDataDtos) {
            GrrViewDataDto grrViewDataDto1 = new GrrViewDataDto();
            BeanUtils.copyProperties(grrViewDataDto, grrViewDataDto1);
            List<KeyValueDto> keyValueDtos = grrViewDataDto1.getColValueDtoList();
            if (keyValueDtos != null && !keyValueDtos.isEmpty()) {
                List<KeyValueDto> keyValueDtos1 = Lists.newLinkedList();
                for (KeyValueDto keyValueDto : keyValueDtos) {
                    if (keyValueDto.getKey().equals(itemName)) {
                        keyValueDtos1.add(keyValueDto);
                        break;
                    }
                }
                grrViewDataDto1.setColValueDtoList(keyValueDtos1);
                grrViewDataDtos.add(grrViewDataDto1);
                keyValueDtos = null;
                grrViewDataDto1 = null;
            }
        }
        return grrViewDataDtos;
    }

    private List<String> getSourceTestItemNames(List<String> filePaths, String scriptPath) {
        PythonUtil pythonUtil = new PythonUtil();
        List<String> includeValue = Lists.newArrayList();
        includeValue.add("-");
        includeValue.add("_");
        includeValue.add("N/A");
        includeValue.add("nil");
        includeValue.add("NaN");
        List<String> itemNames = pythonUtil.getColumnKeys(filePaths, scriptPath, includeValue);
        return itemNames;
    }

    private List<String> getItemNames(List<String> itemNameCheckedLists, int startIndex, int count) {
        List<String> itemNameList = Lists.newLinkedList();
        for (int i = 0; i < count; i++) {
            try {
                itemNameList.add(itemNameCheckedLists.get(startIndex + i));
            } catch (IndexOutOfBoundsException e) {
                break;
            }
        }
        return itemNameList;
    }

    private List<String> getTestItemNames(GrrItemDto grrItemDto, String itemNameFilePath, int startIndex, int endIndex) {
        List<String> itemNameCheckedLists = Lists.newLinkedList();
        FileWriter fw = null;
        try {
            fw = new FileWriter(itemNameFilePath);
            Map<String, Boolean> itemNames = mapper.fromJson(grrItemDto.getTestItemNames(), mapper.contructMapType(Map.class, String.class, Boolean.class));
            if (itemNames != null && !itemNames.isEmpty()) {
                for (String itemName : itemNames.keySet()) {
                    if (itemNames.get(itemName)) {
                        itemNameCheckedLists.add(itemName);
                    }
                }
            }

            if (endIndex <= startIndex && startIndex != -1) {
                return Lists.newLinkedList();
            } else if (startIndex >= 0 && itemNameCheckedLists.size() < endIndex && endIndex != 0) {
                itemNameCheckedLists = itemNameCheckedLists.subList(startIndex, itemNameCheckedLists.size());
            } else if (startIndex >= 0 && itemNameCheckedLists.size() >= endIndex && endIndex != 0) {
                itemNameCheckedLists = itemNameCheckedLists.subList(startIndex, endIndex);
            }

            if (itemNameCheckedLists != null && !itemNameCheckedLists.isEmpty()) {
                for (String name : itemNameCheckedLists) {
                    fw.write(name + System.getProperty("line.separator"));
                }
            }
            itemNames = null;
        } catch (IOException e) {
            logger.error("Get test item error!", e);
            throw new ApplicationException(ExceptionMessages.ERR_20001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GLOBAL_SYSTEM));
        } finally {
            try {
                if (fw != null) {
                    fw.flush();
                    fw.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return itemNameCheckedLists;
    }

    private List<GrrExportResultDto> searchGrrSummaryAndDetail(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey, GrrConfigDto grrConfigDto, GrrItemDto grrItemDto,
                                                               List<String> itemNames, Map<String, List<String>> testDatas, Map<String, Object> uslAndLslAndUnits) {
        if (grrParamDto == null || StringUtils.isBlank(grrParamDto.getFilePath()) || groupSearchKey == null || StringUtils.isBlank(groupSearchKey.getGroupkey1())) {
            throw new ApplicationException(ExceptionMessages.ERR_12001, ResourceBundleUtils.getString(ExceptionMessages.EXCEPTION_GRR_PARAMETER_INVALID));
        }
        List<GrrExportResultDto> grrExportResultDtoList = Lists.newLinkedList();
        groupSearchKey = parseGrrGroup(groupSearchKey);
        grrParamDto.setItemName(null);

        List<GrrViewDataDto> grrViewDataDtos = this.getGrrViewDatas(grrParamDto, groupSearchKey, itemNames, grrConfigDto, testDatas);
        List<GrrSummaryDto> grrSummaryDtoList = this.getGrrSummary(grrParamDto, grrViewDataDtos, uslAndLslAndUnits);

        DateTimeFormatter format1 = DateTimeFormat.forPattern("yyyy/MM/dd");
        DateTime now = new DateTime();
        String time = now.toString(format1);

        String loginName = "";
        Set<String> userKeys = userDataCacheFactory.getCacheUserData().asMap().keySet();
        if (userKeys != null && !userKeys.isEmpty()) {
            for (String userName : userKeys) {
                loginName = userName;
                break;
            }
            userKeys = null;
        }

        int j = 0;
        boolean isHasSubTestItems = false;
        String[] charts = {AppConstant.CHART_TYPE_GRR_XBAR, AppConstant.CHART_TYPE_GRR_R, AppConstant.CHART_TYPE_GRR_RESPONSE_OPERATOR, AppConstant.CHART_TYPE_GRR_RESPONSE,
                AppConstant.CHART_TYPE_GRR_CONTRIBUTION, AppConstant.CHART_TYPE_GRR_INTERACTION};
        String operator = grrParamDto.getOperatorValue();
        String sn = grrParamDto.getPartValue();
        String trial = grrParamDto.getTrialValue();
        String basePath = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_FIELD_PATH).getValue();
        for (int i = 0; i < grrSummaryDtoList.size(); i++) {
            String grr = grrSummaryDtoList.get(i).getGrr();
            if (grr == null || (AppConstant.GLOBAL_NAN).equals(grr) || ("Infinity").equals(grr) || grr.equals("-")) {
                continue;
            }
            j++;
            isHasSubTestItems = true;
            GrrExportParamDto grrExportParamDto = new GrrExportParamDto();
            grrExportParamDto.setItemName(grrSummaryDtoList.get(i).getItemName());
            grrExportParamDto.setUsl(grrSummaryDtoList.get(i).getUsl());
            grrExportParamDto.setLsl(grrSummaryDtoList.get(i).getLsl());
            grrExportParamDto.setTolerance(grrSummaryDtoList.get(i).getTolerance());
            grrExportParamDto.setOperatorCount(operator);
            grrExportParamDto.setSnCount(sn);
            grrExportParamDto.setTrialCount(trial);
            grrExportParamDto.setExportTime(time);
            grrExportParamDto.setPerformedBy(loginName);

            GrrExportResultDto grrExportResultDto = new GrrExportResultDto();
            grrParamDto.setItemName(grrSummaryDtoList.get(i).getItemName());

            List<GrrViewDataDto> grrViewResult = this.getGrrViewDataForItemName(grrParamDto.getItemName(), grrViewDataDtos);
            GrrResultDto grrResultDto = this.searchGrrResultForExport(grrParamDto, grrViewResult, uslAndLslAndUnits);

            grrViewResult = null;
            grrExportResultDto.setGrrViewDataDtos(grrResultDto.getGrrViewDataDtoList());
            grrExportResultDto.setGrrAnovaDtoList(grrResultDto.getGrrAnovaDtoList());
            grrExportResultDto.setGrrSourceDtoList(grrResultDto.getGrrSourceDtoList());
            grrExportResultDto.setGrrStatisticDataDtoList(grrResultDto.getGrrStatisticDataDtoList());
            grrExportResultDto.setGrrExportParamDto(grrExportParamDto);
            grrExportResultDto.setGrrResultDto(grrResultDto);
            if (j == 1) {
                grrExportResultDto.setGrrSummaryDtoList(grrSummaryDtoList);
            }
            grrExportParamDto = null;
            grrResultDto = null;
            List<String> imageList = Lists.newLinkedList();
            for (int k = 0; k < charts.length; k++) {
                StringBuilder sb1 = new StringBuilder();
                sb1.append("export_").append("_").append("no_").append(k).append(charts[k])
                        .append("_").append(System.currentTimeMillis()).append(".").append("gif");
                String imgPath = null;
                if (StringUtils.isNotBlank(basePath)) {
                    imgPath = basePath + "/grr/temp/";
                } else {
                    imgPath = ApplicationPathUtil.getCanonicalPath() + "export/grr/temp/";
                }
                File file = new File(imgPath);
                if (!file.exists()) {
                    file.mkdirs();
                }
                imgPath += sb1.toString();
                imageList.add(imgPath);
            }
            grrExportResultDto.setImgPaths(imageList);
            grrExportResultDtoList.add(grrExportResultDto);
            grrExportResultDto = null;
            imageList = null;
        }

        if (!isHasSubTestItems) {
            GrrExportResultDto grrExportResultDto = new GrrExportResultDto();
            GrrExportParamDto grrExportParamDto = new GrrExportParamDto();
            grrExportParamDto.setOperatorCount(operator);
            grrExportParamDto.setSnCount(sn);
            grrExportParamDto.setTrialCount(trial);
            grrExportParamDto.setExportTime(time);
            grrExportParamDto.setPerformedBy(loginName);
            grrExportResultDto.setGrrSummaryDtoList(grrSummaryDtoList);
            grrExportResultDto.setGrrExportParamDto(grrExportParamDto);
            grrExportResultDtoList.add(grrExportResultDto);
            grrExportResultDto = null;
            grrExportParamDto = null;
        }
        charts = null;
        grrSummaryDtoList = null;
        grrViewDataDtos = null;
        return grrExportResultDtoList;
    }

    private boolean exportSummaryAndDetail(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey, List<String> itemNames, String[] exportFilePaths, ProcessHelper processHelper, ExportThreadManager exportThreadManager) {
        try {
            //set path
            DateTimeFormatter format = DateTimeFormat.forPattern("yyyyMMddHHmmss");
            DateTime now = new DateTime();
            String fix = now.toString(format);
            StringBuilder sb2 = new StringBuilder();
            sb2.append("GRR_Result_Data").append("_").append(fix);
            String fileName = sb2.toString();
            String savePath = null;
            processHelper.postProcess(12, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_DISENV));
            String basePath = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_FIELD_PATH).getValue();
            if (StringUtils.isNotBlank(basePath)) {
                savePath = basePath + "/grr/";
            } else {
                savePath = ApplicationPathUtil.getCanonicalPath() + "export/grr/";
            }
            File file = new File(savePath);
            if (!file.exists()) {
                file.mkdirs();
            }

            processHelper.postProcess(savePath);
            exportFilePaths[0] = savePath + fileName;
            String exportFilePath = exportFilePaths[0];
            processHelper.postProcess(18, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CACHE));
            logger.debug("Export grr result and data to file:{}", fileName);
            String paramFilePath = grrParamDto.getFilePath();
            String name = paramFilePath.substring(paramFilePath.lastIndexOf(File.separator) + 1, paramFilePath.length());
            //get required configuration
            GrrConfigDto grrConfigDto = systemService.findGrrConfig();
            GrrItemDto grrItemDto = systemService.findGrrItemByGrrName(name);
            SystemConfigDto systemConfigDto = systemService.findSystemConfig();

            //validate and set params
            validateParams(grrConfigDto, grrItemDto, grrParamDto, systemConfigDto);

            List<String> filePaths = Lists.newArrayList();
            filePaths.add(paramFilePath);
            String scriptPath = this.getScriptPath();

            //get usl, lsl, unit the all testItem
            Map<String, Object> uslAndLslAndUnits = this.getUslAndLslAndUnits(filePaths, scriptPath);

            int readPieceSize = AppConstant.DEFAULT_COUNT;
            if (systemConfigDto != null) {
                readPieceSize = systemConfigDto.getGrrExportNumber();
            }

            processHelper.postProcess(20, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_START_EXPORT));
            ChartExporterImpl exporter = new ChartExporterImpl(AppConstant.ANALYSIS_GRR_DIGNUM);
            int excelIndex = 0;
            Map<String, Object> exportParams = Maps.newHashMap();
            if (itemNames != null && !itemNames.isEmpty()) {
                final double process20 = 20.0;
                final double process80 = 80.0;
                double progress = (process80 / (Math.ceil((double) itemNames.size() / readPieceSize)) == process80 ? process20 + process20 : (process80 / (Math.ceil((double) itemNames.size() / readPieceSize))));
                int process = (int) process20;
                for (int startIndex = 0; startIndex < itemNames.size(); startIndex += readPieceSize) {
                    if (!this.isNullExportThread(exportThreadManager)) {
                        return false;
                    }
                    excelIndex++;

                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DATA));
                    List<String> itemNameResult = getItemNames(itemNames, startIndex, readPieceSize);
                    Map<String, List<String>> testDatas = this.getDataByName(filePaths, scriptPath, itemNameResult);
                    List<GrrExportResultDto> grrExportResultDtoList = searchGrrSummaryAndDetail(grrParamDto, groupSearchKey, grrConfigDto, grrItemDto, itemNameResult, testDatas, uslAndLslAndUnits);

                    exportParams.put("grrName", grrParamDto.getGrrName());
                    exportParams.put("sigma", grrParamDto.getSigma());
                    exportParams.put("rule", grrParamDto.getRule());
                    exportParams.put("model", AppConstant.GRR_MODEL_NORMAL);
                    exportParams.put("digGrrNum", grrParamDto.getDigitNum());

                    int operator = Integer.parseInt(grrParamDto.getOperatorValue());
                    int trial = Integer.parseInt(grrParamDto.getTrialValue());

                    List<GrrSummaryDto> grrSummaryDtoList = Lists.newArrayList();
                    if (!grrExportResultDtoList.isEmpty()) {
                        grrSummaryDtoList = grrExportResultDtoList.get(0).getGrrSummaryDtoList();
                    }
                    for (int i = 0; i < grrExportResultDtoList.size(); i++) {
                        if (!this.isNullExportThread(exportThreadManager)) {
                            return false;
                        }
                        GrrExportResultDto grrExportResultDto = grrExportResultDtoList.get(i);
                        exporter.exportGrrChart(grrExportResultDto.getGrrResultDto(), grrExportResultDto.getGrrExportParamDto(), grrExportResultDto.getImgPaths());
                        grrExportResultDto = null;
                    }

                    if (!this.isNullExportThread(exportThreadManager)) {
                        return false;
                    }

                    GrrExportWorker factory = new GrrExportWorker();
                    factory.buildSummaryAndDetail(exportParams, grrSummaryDtoList, grrExportResultDtoList);
                    GrrExcelBuilder grrExcelBuilder = new GrrExcelBuilder(operator, trial);

                    FileUtils.createDir(exportFilePath);
                    String filePath = exportFilePath + "/" + "GRR_Result_Data" + "_" + fix + "_" + excelIndex + ".xlsx";

                    if (!this.isNullExportThread(exportThreadManager)) {
                        return false;
                    }

                    grrExcelBuilder.drawExcel(filePath, factory);
                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CREATE_FILE) + fileName + "_" + excelIndex);
                    logger.debug("Export grr result and data to filepath:{}", exportFilePath);

                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE));
                    String imageDir = savePath + "/temp";
                    FileUtils.deleteDir(imageDir);
                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CLEAN_CACHE_DONE));

                    factory = null;
                    exportParams.clear();
                    grrSummaryDtoList = null;
                    grrExportResultDtoList = null;
                    grrExcelBuilder = null;
                    itemNameResult = null;
                    testDatas = null;
                    processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DONE));
                    process += progress;
                }
                String fileName1 = name.substring(0, name.lastIndexOf(".")) + "_" + fix + ".csv";
                FileUtil.copyFile(grrParamDto.getFilePath(), exportFilePath + "/" + fileName1);
                uslAndLslAndUnits = null;
                exporter = null;
            }
        } catch (ApplicationException e) {
            logger.error("Export grr result and data failed!", e);
            throw new ApplicationException(e.getCode(), e.getMessage());
        }
        return true;
    }

    private boolean exportSummary(GrrParamDto grrParamDto, GrrGroupDto groupSearchKey, List<String> itemNames, String[] exportFilePaths, ProcessHelper processHelper, ExportThreadManager exportThreadManager) {
        try {
            //set path
            DateTimeFormatter format = DateTimeFormat.forPattern("yyyyMMddHHmmss");
            DateTime now = new DateTime();
            String fix = now.toString(format);
            StringBuilder sb2 = new StringBuilder();
            sb2.append("GRR_Result").append("_").append(fix);
            String fileName = sb2.toString();
            String savePath = null;
            String basePath = sysConfigService.findByName(AppConstant.SYSTEM_CONFIG_FIELD_PATH).getValue();
            if (StringUtils.isNotBlank(basePath)) {
                savePath = basePath + "/grr/";
            } else {
                savePath = ApplicationPathUtil.getCanonicalPath() + "export/grr/";
            }
            File file = new File(savePath);
            if (!file.exists()) {
                file.mkdirs();
            }

            processHelper.postProcess(savePath);
            exportFilePaths[0] = savePath + fileName;
            String exportFilePath = exportFilePaths[0];
            logger.debug("Export grr only result to file:{}", fileName);

            Map<String, Object> exportParams = Maps.newHashMap();
            String paramFilePath = grrParamDto.getFilePath();
            String name = paramFilePath.substring(paramFilePath.lastIndexOf(File.separator) + 1, paramFilePath.length());

            //get required configuration
            GrrConfigDto grrConfigDto = systemService.findGrrConfig();
            GrrItemDto grrItemDto = systemService.findGrrItemByGrrName(name);
            SystemConfigDto systemConfigDto = systemService.findSystemConfig();

            //validate and set params
            validateParams(grrConfigDto, grrItemDto, grrParamDto, systemConfigDto);

            List<String> filePaths = Lists.newArrayList();
            filePaths.add(paramFilePath);
            String scriptPath = this.getScriptPath();

            //get usl, lsl, unit the all testItem
            Map<String, Object> uslAndLslAndUnits = this.getUslAndLslAndUnits(filePaths, scriptPath);

            processHelper.postProcess(18, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DATA));
            int readPieceSize = AppConstant.DEFAULT_COUNT;
            if (systemConfigDto != null) {
                readPieceSize = systemConfigDto.getGrrExportNumber();
            }
            List<GrrSummaryDto> grrSummaryDtoList = Lists.newLinkedList();
            if (itemNames != null && !itemNames.isEmpty()) {
                GrrExportWorker factory = new GrrExportWorker();
                final double process20 = 20.0;
                final double process80 = 80.0;
                double progress = (process80 / (Math.ceil((double) itemNames.size() / readPieceSize)) == process80 ? (process20 + process20) : (process80 / (Math.ceil((double) itemNames.size() / readPieceSize))));
                int process = (int) process20;
                for (int startIndex = 0; startIndex < itemNames.size(); startIndex += readPieceSize) {
                    if (!this.isNullExportThread(exportThreadManager)) {
                        return false;
                    }
                    List<String> itemNameResult = getItemNames(itemNames, startIndex, readPieceSize);
                    Map<String, List<String>> testDatas = this.getDataByName(filePaths, scriptPath, itemNameResult);
                    List<GrrViewDataDto> grrViewDataDtos = this.getGrrViewDatas(grrParamDto, groupSearchKey, itemNameResult, grrConfigDto, testDatas);
                    grrSummaryDtoList.addAll(this.getGrrSummary(grrParamDto, grrViewDataDtos, uslAndLslAndUnits));
                    exportParams.put("grrName", grrParamDto.getGrrName());
                    exportParams.put("rule", grrParamDto.getRule());
                    exportParams.put("digGrrNum", grrParamDto.getDigitNum());
                    factory.buildGrrSummary(exportParams, grrSummaryDtoList, startIndex);
                    grrViewDataDtos = null;
                    testDatas = null;
                    itemNameResult = null;
                    grrSummaryDtoList.clear();
                    processHelper.postProcess(process, "");
                    process += progress;
                }

                GrrExcelBuilder grrExcelBuilder = new GrrExcelBuilder(null, null);
                FileUtils.createDir(exportFilePath);
                logger.debug("Export grr only result to filepath:{}", exportFilePath);
                String filePath = exportFilePath + "/" + fileName + ".xlsx";
                grrExcelBuilder.drawExcel(filePath, factory);
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_WRITE_DONE));
                processHelper.postProcess(process, ResourceBundleUtils.getString(AppConstant.LOG_EXPORT_FILE_CREATE_FILE) + fileName);
                factory = null;
                exportParams.clear();
                grrSummaryDtoList = null;
                grrExcelBuilder = null;
                uslAndLslAndUnits = null;
                String fileName1 = name.substring(0, name.lastIndexOf(".")) + "_" + fix + ".csv";
                FileUtil.copyFile(grrParamDto.getFilePath(), exportFilePath + "/" + fileName1);
            }
        } catch (ApplicationException e) {
            logger.error("Export grr result and data failed!", e);
            throw new ApplicationException(e.getCode(), e.getMessage());
        }
        return true;
    }

    private boolean isNullExportThread(ExportThreadManager exportThreadManager) {
        if (exportThreadManager != null) {
            Thread exportThread = exportThreadManager.getGrrExportThreads().get(Thread.currentThread().getName());
            if (exportThread == null) {
                return false;
            }
        }
        return true;
    }
}
